lib.py

import pprint
import re
import urllib
from tempfile import NamedTemporaryFile

import numpy as np
from PIL import Image

import storage


def get_modes(storage):
    return storage.modes()


def get_scalar_tags(storage, mode):
    result = {}
    for mode in storage.modes():
        reader = storage.as_mode(mode)
        tags = reader.tags('scalar')
        if tags:
            result[mode] = {}
            for tag in tags:
                result[mode][tag] = {
                    'displayName': reader.scalar(tag).caption(),
                    'description': "",
                }
    return result


def get_scalar(storage, mode, tag):
    reader = storage.as_mode(mode)
    scalar = reader.scalar(tag)

    records = scalar.records()
    ids = scalar.ids()
    timestamps = scalar.timestamps()

    result = zip(timestamps, ids, records)
    return result


def get_image_tags(storage):
    result = {}

    for mode in storage.modes():
        reader = storage.as_mode(mode)
        tags = reader.tags('image')
        if tags:
            result[mode] = {}
            for tag in tags:
                image = reader.image(tag)
                for i in xrange(max(1, image.num_samples())):
                    caption = tag if image.num_samples() <= 1 else '%s/%d'%(tag, i)
                    result[mode][caption] = {
                        'displayName': caption,
                        'description': "",
                        'samples': 1,
                    }
    return result


def get_image_tag_steps(storage, mode, tag):
    print 'image_tag_steps,mode,tag:', mode, tag
    # remove suffix '/x'
    res = re.search(r".*/([0-9]+$)", tag)
    sample_index = 0
    origin_tag = tag
    if res:
        tag = tag[:tag.rfind('/')]
        sample_index = int(res.groups()[0])

    reader = storage.as_mode(mode)
    image = reader.image(tag)
    res = []

    for step_index in range(image.num_records()):
        record = image.record(step_index, sample_index)
        shape = record.shape()
        assert shape, "%s,%s" % (mode, tag)
        query = urllib.urlencode({
            'sample': 0,
            'index': step_index,
            'tag': origin_tag,
            'run': mode,
        })
        res.append({
            'height': shape[0],
            'width': shape[1],
            'step': record.step_id(),
            'wall_time': image.timestamp(step_index),
            'query': query,
        })
    return res


def get_invididual_image(storage, mode, tag, step_index):
    reader = storage.as_mode(mode)
    res = re.search(r".*/([0-9]+$)", tag)
    # remove suffix '/x'
    if res:
        offset = int(res.groups()[0])
        tag = tag[:tag.rfind('/')]

    image = reader.image(tag)
    record = image.record(step_index, offset)

    data = np.array(record.data(), dtype='uint8').reshape(record.shape())
    tempfile = NamedTemporaryFile(mode='w+b', suffix='.png')
    with Image.fromarray(data) as im:
        im.save(tempfile)
    tempfile.seek(0, 0)
    return tempfile


if __name__ == '__main__':
    reader = storage.StorageReader('./tmp/mock')
    tags = get_image_tags(reader)

    tags = get_image_tag_steps(reader, 'train', 'layer1/layer2/image0/0')
    pprint.pprint(tags)

    image = get_invididual_image(reader, "train", 'layer1/layer2/image0/0', 2)
    print image