Forum Archive

I tried to use MNIST with this code below, but it does not work.
How Can I use MNIST with Pythonista?

Thank you.

# coding: utf-8
try:
    import urllib.request
except ImportError:
    raise ImportError('You should use Python 3.x')
import os.path 
import os 
import numpy as np 


url_base = 'http://yann.lecun.com/exdb/mnist/' 
key_file = {
    'train_img':'train-images-idx3-ubyte.gz',
    'train_label':'train-labels-idx1-ubyte.gz',
    'test_img':'t10k-images-idx3-ubyte.gz',
    'test_label':'t10k-labels-idx1-ubyte.gz'
}

dataset_dir = os.path.dirname(os.path.abspath(__file__)) 
save_file = dataset_dir + "/mnist.pkl" 

train_num = 60000
test_num = 10000
img_dim = (1, 28, 28)
img_size = 784


def _download(file_name):
    file_path = dataset_dir + "/" + file_name

    if os.path.exists(file_path):
        return

    print("Downloading " + file_name + " ... ")
    urllib.request.urlretrieve(url_base + file_name, file_path)
    print("Done")

def download_mnist():
    for v in key_file.values():
        _download(v)

def _load_label(file_name):
    file_path = dataset_dir + "/" + file_name

    print("Converting " + file_name + " to NumPy Array ...")
    with gzip.open(file_path, 'rb') as f:
            labels = np.frombuffer(f.read(), np.uint8, offset=8)
    print("Done")

    return labels

def _load_img(file_name):
    file_path = dataset_dir + "/" + file_name

    print("Converting " + file_name + " to NumPy Array ...")    
    with gzip.open(file_path, 'rb') as f:
            data = np.frombuffer(f.read(), np.uint8, offset=16)
    data = data.reshape(-1, img_size)
    print("Done")

    return data

def _convert_numpy():
    dataset = {}
    dataset['train_img'] =  _load_img(key_file['train_img'])
    dataset['train_label'] = _load_label(key_file['train_label'])    
    dataset['test_img'] = _load_img(key_file['test_img'])
    dataset['test_label'] = _load_label(key_file['test_label'])

    return dataset

def init_mnist():
    download_mnist()
    dataset = _convert_numpy()
    (x_train, t_train),(x_test, t_test) = load_mnist()
    img = x_train[0]
    pil_img = Image.fromarray(np.uint8(img))
    pil_img.show()
    print("Creating pickle file ...")
    with open(save_file, 'wb') as f:
        pickle.dump(dataset, f, -1)
    print("Done!")

def _change_ont_hot_label(X):
    T = np.zeros((X.size, 10))
    for idx, row in enumerate(T):
        row[X[idx]] = 1

    return T


def load_mnist(normalize=True, flatten=True, one_hot_label=False):
    if not os.path.exists(save_file):
        init_mnist()

    with open(save_file, 'rb') as f:
        dataset = pickle.load(f)

    if normalize:
        for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].astype(np.float32)
            dataset[key] /= 255.0

    if one_hot_label:
        dataset['train_label'] = _change_ont_hot_label(dataset['train_label'])
        dataset['test_label'] = _change_ont_hot_label(dataset['test_label'])    

    if not flatten:
         for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].reshape(-1, 1, 28, 28)

    return (dataset['train_img'], dataset['train_label']), (dataset['test_img'], dataset['test_label']) 


if __name__ == '__main__':
    init_mnist()

Add the line:

import gzip

It will still crash but you get one step further.

Init_mnist() and load_mnist() call each other in an infinite loop.

These following lines will also be useful as you further debug this code.

import pickle
from PIL import Image

Did you write this or get it from elsewhere?

Thank you.

I got this from a book describing about deep learning, but I edited a little.
I'm a beginner about Python and I only have an iPad now.

I searched another way to use MNIST but I don't understand how it works really.

I didn't noticed I accidentally deleted 2 lines when uploaded here.

import gzip 
import pickle

https://github.com/sorki/python-mnist is worth looking at.

What is the title of the book? Maybe I will read that too.

Thanks. but I give up...
I will try Installing MNIST on my Mac later.

The book is quite useful but only in Japanese.
Describing about deep learning only with NumPy and Matplotlib.

ゼロから作るDeep Learning ―Pythonで学ぶディープラーニングの理論と実装 斎藤 康毅 https://www.amazon.co.jp/dp/4873117585/ref=cm_sw_r_tw_dp_x_9vnJyb8SQD70V

Is there a URL to the source code that is in that book?

Perhaps here found by Googleing some lines of the code

Thanks @cvp

In fact, the problem is in Pythonista... https://github.com/omz/Pythonista-Issues/issues/260