【ひらがな認識AI①】データをゲットして使える型に変換しよう

# ライブラリのインポート
from zipfile import ZipFile
import struct
import numpy as np
from PIL import Image

DATA_PATH = ###--zipファイルがあるディレクトリパス--###
RECORD_SIZE = 8199
etl = []
info = []

with ZipFile(DATA_PATH + "ETL8G.zip") as etl1:
    names = [n for n in etl1.namelist() if "_" in n]
    for x in names:
        with etl1.open(x) as f:
            while True:
                s = f.read(RECORD_SIZE)
                if (s is None) or (len(s) < RECORD_SIZE):
                    break
                r = struct.unpack(">HH8sIBBBBHHHHBB30x8128s11x", s)
                img = Image.frombytes("F", (128, 127), r[14], "bit", (4, 0))
                img = np.array(img.convert("L"))  # 0..15
                lbl = r[1]
                if lbl in range(int(0x2420), int(0x2474)):  # 平仮名のみ
                    etl.append((img, lbl))
                    info.append(r[:-1])

# 以下のお二方のコードを参考にさせていただきました。
# kcrtさん     https://qiita.com/kcrt/items/a7f0582a91d6599d164d
# 奥村晴彦さん  https://okumuralab.org/~okumura/python/etlcdb.html

print(etl[0])

# (array([[0, 0, 0, ..., 0, 0, 0],
#       [0, 0, 0, ..., 0, 0, 0],
#       [0, 0, 0, ..., 0, 0, 0],
#       ...,
#       [0, 0, 0, ..., 0, 0, 0],
#       [0, 0, 0, ..., 0, 0, 0],
#       [0, 0, 0, ..., 0, 0, 0]], dtype=uint8), 9250)

print(info[0])

# (1, 9250, b'A.HIRA  ', 1, 0, 0, 1, 24, 3552, 0, 8001, 16880, 0, 0)

import matplotlib.pyplot as plt

plt.imshow(etl[0][0], cmap="gray_r")
plt.show()

import os

ar = []
with open(DATA_PATH + "JIS0208.TXT") as f:  # DATA_PATHに変換表を配置
    for t_line in f:
        if t_line[0] != "#":
            sjis, jis, utf16 = os.path.basename(t_line).split("\t")[0:3]
            ar.append([jis, utf16])
ar = dict(ar)

def decoder(x):
    x = str(hex(x))[2:]
    return chr(int(ar["0x"+x.upper()], 16))

# こちらの方のコードを参考にさせていただきました。
# hinomaさん  https://qiita.com/hinoma/items/9e96b25eeeee01437c88

print(decoder(etl[0][1]))

# あ

import pandas as pd

col_names = ["sheet","jis_code","reading","id","eval_ind",
             "eval_group","sex","age","id_industrial","occupation",
             "date_sheet","date_scan","x","y"]

df = pd.DataFrame(info, columns=col_names)
df["reading"] = df["reading"].apply(lambda x: x.decode())
df["label"] = df["jis_code"].apply(decoder)  # 文字コードを文字に変換

import japanize_matplotlib  # 日本語を表示させるのに必要

perm = np.random.permutation(range(len(etl)))
for k in range(24):
    plt.subplot(3, 8, k+1)
    plt.xticks([])
    plt.yticks([])
    img, lbl = etl[perm[k]]
    plt.imshow(img>=4, cmap="gray_r")
    plt.title(decoder(lbl))
plt.show()

# こちらの方のコードをほぼそのままパクりました。
# 奥村晴彦さん  https://okumuralab.org/~okumura/python/etlcdb.html