# -*- coding: utf-8 -*- ''' General documentation architecture: Home Index - Getting started Getting started with the sequential model Getting started with the functional api FAQ - Models About Keras models explain when one should use Sequential or functional API explain compilation step explain weight saving, weight loading explain serialization, deserialization Sequential Model (functional API) - Layers About Keras layers explain common layer functions: get_weights, set_weights, get_config explain input_shape explain usage on non-Keras tensors Core Layers Convolutional Layers Pooling Layers Locally-connected Layers Recurrent Layers Embedding Layers Merge Layers Advanced Activations Layers Normalization Layers Noise Layers Layer Wrappers Writing your own Keras layers - Preprocessing Sequence Preprocessing Text Preprocessing Image Preprocessing Losses Metrics Optimizers Activations Callbacks Datasets Applications Backend Initializers Regularizers Constraints Visualization Scikit-learn API Utils Contributing ''' from __future__ import print_function from __future__ import unicode_literals import re import inspect import os import shutil import keras from keras import utils from keras import layers from keras.layers import advanced_activations from keras.layers import noise from keras.layers import wrappers from keras import initializers from keras import optimizers from keras import callbacks from keras import models from keras import losses from keras import metrics from keras import backend from keras import activations from keras import preprocessing import sys if sys.version[0] == '2': reload(sys) sys.setdefaultencoding('utf8') EXCLUDE = { 'Optimizer', 'Wrapper', 'get_session', 'set_session', 'CallbackList', 'serialize', 'deserialize', 'get', 'set_image_dim_ordering', 'image_dim_ordering', 'get_variable_shape', } # For each class to document, it is possible to: # 1) Document only the class: [classA, classB, ...] # 2) Document all its methods: [classA, (classB, "*")] # 3) Choose which methods to document (methods listed as strings): # [classA, (classB, ["method1", "method2", ...]), ...] # 4) Choose which methods to document (methods listed as qualified names): # [classA, (classB, [module.classB.method1, module.classB.method2, ...]), ...] PAGES = [ { 'page': 'models/sequential.md', 'functions': [ models.Sequential.compile, models.Sequential.fit, models.Sequential.evaluate, models.Sequential.predict, models.Sequential.train_on_batch, models.Sequential.test_on_batch, models.Sequential.predict_on_batch, models.Sequential.fit_generator, models.Sequential.evaluate_generator, models.Sequential.predict_generator, models.Sequential.get_layer, ], }, { 'page': 'models/model.md', 'functions': [ models.Model.compile, models.Model.fit, models.Model.evaluate, models.Model.predict, models.Model.train_on_batch, models.Model.test_on_batch, models.Model.predict_on_batch, models.Model.fit_generator, models.Model.evaluate_generator, models.Model.predict_generator, models.Model.get_layer, ] }, { 'page': 'layers/core.md', 'classes': [ layers.Dense, layers.Activation, layers.Dropout, layers.Flatten, layers.Input, layers.Reshape, layers.Permute, layers.RepeatVector, layers.Lambda, layers.ActivityRegularization, layers.Masking, layers.SpatialDropout1D, layers.SpatialDropout2D, layers.SpatialDropout3D, ], }, { 'page': 'layers/convolutional.md', 'classes': [ layers.Conv1D, layers.Conv2D, layers.SeparableConv1D, layers.SeparableConv2D, layers.Conv2DTranspose, layers.Conv3D, layers.Cropping1D, layers.Cropping2D, layers.Cropping3D, layers.UpSampling1D, layers.UpSampling2D, layers.UpSampling3D, layers.ZeroPadding1D, layers.ZeroPadding2D, layers.ZeroPadding3D, ], }, { 'page': 'layers/pooling.md', 'classes': [ layers.MaxPooling1D, layers.MaxPooling2D, layers.MaxPooling3D, layers.AveragePooling1D, layers.AveragePooling2D, layers.AveragePooling3D, layers.GlobalMaxPooling1D, layers.GlobalAveragePooling1D, layers.GlobalMaxPooling2D, layers.GlobalAveragePooling2D, layers.GlobalMaxPooling3D, layers.GlobalAveragePooling3D, ], }, { 'page': 'layers/local.md', 'classes': [ layers.LocallyConnected1D, layers.LocallyConnected2D, ], }, { 'page': 'layers/recurrent.md', 'classes': [ layers.RNN, layers.SimpleRNN, layers.GRU, layers.LSTM, layers.ConvLSTM2D, layers.SimpleRNNCell, layers.GRUCell, layers.LSTMCell, layers.CuDNNGRU, layers.CuDNNLSTM, ], }, { 'page': 'layers/embeddings.md', 'classes': [ layers.Embedding, ], }, { 'page': 'layers/normalization.md', 'classes': [ layers.BatchNormalization, ], }, { 'page': 'layers/advanced-activations.md', 'all_module_classes': [advanced_activations], }, { 'page': 'layers/noise.md', 'all_module_classes': [noise], }, { 'page': 'layers/merge.md', 'classes': [ layers.Add, layers.Subtract, layers.Multiply, layers.Average, layers.Maximum, layers.Concatenate, layers.Dot, ], 'functions': [ layers.add, layers.subtract, layers.multiply, layers.average, layers.maximum, layers.concatenate, layers.dot, ] }, { 'page': 'preprocessing/sequence.md', 'functions': [ preprocessing.sequence.pad_sequences, preprocessing.sequence.skipgrams, preprocessing.sequence.make_sampling_table, ], 'classes': [ preprocessing.sequence.TimeseriesGenerator, ] }, { 'page': 'preprocessing/image.md', 'classes': [ (preprocessing.image.ImageDataGenerator, '*') ] }, { 'page': 'preprocessing/text.md', 'functions': [ preprocessing.text.hashing_trick, preprocessing.text.one_hot, preprocessing.text.text_to_word_sequence, ], 'classes': [ preprocessing.text.Tokenizer, ] }, { 'page': 'layers/wrappers.md', 'all_module_classes': [wrappers], }, { 'page': 'metrics.md', 'all_module_functions': [metrics], }, { 'page': 'losses.md', 'all_module_functions': [losses], }, { 'page': 'initializers.md', 'all_module_functions': [initializers], 'all_module_classes': [initializers], }, { 'page': 'optimizers.md', 'all_module_classes': [optimizers], }, { 'page': 'callbacks.md', 'all_module_classes': [callbacks], }, { 'page': 'activations.md', 'all_module_functions': [activations], }, { 'page': 'backend.md', 'all_module_functions': [backend], }, { 'page': 'utils.md', 'functions': [utils.to_categorical, utils.normalize, utils.get_file, utils.print_summary, utils.plot_model, utils.multi_gpu_model], 'classes': [utils.CustomObjectScope, utils.HDF5Matrix, utils.Sequence], }, ] ROOT = 'http://keras.io/' def get_function_signature(function, method=True): wrapped = getattr(function, '_original_function', None) if wrapped is None: signature = inspect.getargspec(function) else: signature = inspect.getargspec(wrapped) defaults = signature.defaults if method: args = signature.args[1:] else: args = signature.args if defaults: kwargs = zip(args[-len(defaults):], defaults) args = args[:-len(defaults)] else: kwargs = [] st = '%s.%s(' % (clean_module_name(function.__module__), function.__name__) for a in args: st += str(a) + ', ' for a, v in kwargs: if isinstance(v, str): v = '\'' + v + '\'' st += str(a) + '=' + str(v) + ', ' if kwargs or args: signature = st[:-2] + ')' else: signature = st + ')' if not method: # Prepend the module name. signature = clean_module_name(function.__module__) + '.' + signature return post_process_signature(signature) def get_class_signature(cls): try: class_signature = get_function_signature(cls.__init__) class_signature = class_signature.replace('__init__', cls.__name__) except (TypeError, AttributeError): # in case the class inherits from object and does not # define __init__ class_signature = clean_module_name(cls.__module__) + '.' + cls.__name__ + '()' return post_process_signature(class_signature) def post_process_signature(signature): parts = re.split(r'\.(?!\d)', signature) if len(parts) >= 4: if parts[1] == 'layers': signature = 'keras.layers.' + '.'.join(parts[3:]) if parts[1] == 'utils': signature = 'keras.utils.' + '.'.join(parts[3:]) if parts[1] == 'backend': signature = 'keras.backend.' + '.'.join(parts[3:]) return signature def clean_module_name(name): if name.startswith('keras_applications'): name = name.replace('keras_applications', 'keras.applications') if name.startswith('keras_preprocessing'): name = name.replace('keras_preprocessing', 'keras.preprocessing') assert name[:6] == 'keras.', 'Invalid module name: %s' % name return name def class_to_docs_link(cls): module_name = clean_module_name(cls.__module__) module_name = module_name[6:] link = ROOT + module_name.replace('.', '/') + '#' + cls.__name__.lower() return link def class_to_source_link(cls): module_name = clean_module_name(cls.__module__) path = module_name.replace('.', '/') path += '.py' line = inspect.getsourcelines(cls)[-1] link = ('https://github.com/keras-team/' 'keras/blob/master/' + path + '#L' + str(line)) return '[[source]](' + link + ')' def code_snippet(snippet): result = '```python\n' result += snippet + '\n' result += '```\n' return result def count_leading_spaces(s): ws = re.search(r'\S', s) if ws: return ws.start() else: return 0 def process_list_block(docstring, starting_point, leading_spaces, marker): ending_point = docstring.find('\n\n', starting_point) block = docstring[starting_point:None if ending_point == -1 else ending_point - 1] # Place marker for later reinjection. docstring = docstring.replace(block, marker) lines = block.split('\n') # Remove the computed number of leading white spaces from each line. lines = [re.sub('^' + ' ' * leading_spaces, '', line) for line in lines] # Usually lines have at least 4 additional leading spaces. # These have to be removed, but first the list roots have to be detected. top_level_regex = r'^ ([^\s\\\(]+):(.*)' top_level_replacement = r'- __\1__:\2' lines = [re.sub(top_level_regex, top_level_replacement, line) for line in lines] # All the other lines get simply the 4 leading space (if present) removed lines = [re.sub(r'^ ', '', line) for line in lines] # Fix text lines after lists indent = 0 text_block = False for i in range(len(lines)): line = lines[i] spaces = re.search(r'\S', line) if spaces: # If it is a list element if line[spaces.start()] == '-': indent = spaces.start() + 1 if text_block: text_block = False lines[i] = '\n' + line elif spaces.start() < indent: text_block = True indent = spaces.start() lines[i] = '\n' + line else: text_block = False indent = 0 block = '\n'.join(lines) return docstring, block def process_docstring(docstring): # First, extract code blocks and process them. code_blocks = [] if '```' in docstring: tmp = docstring[:] while '```' in tmp: tmp = tmp[tmp.find('```'):] index = tmp[3:].find('```') + 6 snippet = tmp[:index] # Place marker in docstring for later reinjection. docstring = docstring.replace( snippet, '$CODE_BLOCK_%d' % len(code_blocks)) snippet_lines = snippet.split('\n') # Remove leading spaces. num_leading_spaces = snippet_lines[-1].find('`') snippet_lines = ([snippet_lines[0]] + [line[num_leading_spaces:] for line in snippet_lines[1:]]) # Most code snippets have 3 or 4 more leading spaces # on inner lines, but not all. Remove them. inner_lines = snippet_lines[1:-1] leading_spaces = None for line in inner_lines: if not line or line[0] == '\n': continue spaces = count_leading_spaces(line) if leading_spaces is None: leading_spaces = spaces if spaces < leading_spaces: leading_spaces = spaces if leading_spaces: snippet_lines = ([snippet_lines[0]] + [line[leading_spaces:] for line in snippet_lines[1:-1]] + [snippet_lines[-1]]) snippet = '\n'.join(snippet_lines) code_blocks.append(snippet) tmp = tmp[index:] # Format docstring lists. section_regex = r'\n( +)# (.*)\n' section_idx = re.search(section_regex, docstring) shift = 0 sections = {} while section_idx and section_idx.group(2): anchor = section_idx.group(2) leading_spaces = len(section_idx.group(1)) shift += section_idx.end() marker = '$' + anchor.replace(' ', '_') + '$' docstring, content = process_list_block(docstring, shift, leading_spaces, marker) sections[marker] = content section_idx = re.search(section_regex, docstring[shift:]) # Format docstring section titles. docstring = re.sub(r'\n(\s+)# (.*)\n', r'\n\1__\2__\n\n', docstring) # Strip all remaining leading spaces. lines = docstring.split('\n') docstring = '\n'.join([line.lstrip(' ') for line in lines]) # Reinject list blocks. for marker, content in sections.items(): docstring = docstring.replace(marker, content) # Reinject code blocks. for i, code_block in enumerate(code_blocks): docstring = docstring.replace( '$CODE_BLOCK_%d' % i, code_block) return docstring print('Cleaning up existing sources directory.') if os.path.exists('sources'): shutil.rmtree('sources') print('Populating sources directory with templates.') for subdir, dirs, fnames in os.walk('templates'): for fname in fnames: new_subdir = subdir.replace('templates', 'sources') if not os.path.exists(new_subdir): os.makedirs(new_subdir) if fname[-3:] == '.md': fpath = os.path.join(subdir, fname) new_fpath = fpath.replace('templates', 'sources') shutil.copy(fpath, new_fpath) def read_file(path): with open(path) as f: return f.read() def collect_class_methods(cls, methods): if isinstance(methods, (list, tuple)): return [getattr(cls, m) if isinstance(m, str) else m for m in methods] methods = [] for _, method in inspect.getmembers(cls, predicate=inspect.isroutine): if method.__name__[0] == '_' or method.__name__ in EXCLUDE: continue methods.append(method) return methods def render_function(function, method=True): subblocks = [] signature = get_function_signature(function, method=method) if method: signature = signature.replace(clean_module_name(function.__module__) + '.', '') else: signature = signature.replace(clean_module_name(function.__module__) + '.', '', 1) subblocks.append('### ' + function.__name__ + '\n') subblocks.append(code_snippet(signature)) docstring = function.__doc__ if docstring: subblocks.append(process_docstring(docstring)) return '\n\n'.join(subblocks) def read_page_data(page_data, type): assert type in ['classes', 'functions'] data = page_data.get(type, []) for module in page_data.get('all_module_{}'.format(type), []): module_data = [] for name in dir(module): if name[0] == '_' or name in EXCLUDE: continue module_member = getattr(module, name) if (inspect.isclass(module_member) and type == 'classes' or inspect.isfunction(module_member) and type == 'functions'): instance = module_member if module.__name__ in instance.__module__: if instance not in module_data: module_data.append(instance) module_data.sort(key=lambda x: id(x)) data += module_data return data if __name__ == '__main__': readme = read_file('../README.md') index = read_file('templates/index.md') index = index.replace('{{autogenerated}}', readme[readme.find('##'):]) with open('sources/index.md', 'w') as f: f.write(index) print('Generating docs for Keras %s.' % keras.__version__) for page_data in PAGES: classes = read_page_data(page_data, 'classes') blocks = [] for element in classes: if not isinstance(element, (list, tuple)): element = (element, []) cls = element[0] subblocks = [] signature = get_class_signature(cls) subblocks.append('' + class_to_source_link(cls) + '') if element[1]: subblocks.append('## ' + cls.__name__ + ' class\n') else: subblocks.append('### ' + cls.__name__ + '\n') subblocks.append(code_snippet(signature)) docstring = cls.__doc__ if docstring: subblocks.append(process_docstring(docstring)) methods = collect_class_methods(cls, element[1]) if methods: subblocks.append('\n---') subblocks.append('## ' + cls.__name__ + ' methods\n') subblocks.append('\n---\n'.join( [render_function(method, method=True) for method in methods])) blocks.append('\n'.join(subblocks)) functions = read_page_data(page_data, 'functions') for function in functions: blocks.append(render_function(function, method=False)) if not blocks: raise RuntimeError('Found no content for page ' + page_data['page']) mkdown = '\n----\n\n'.join(blocks) # save module page. # Either insert content into existing page, # or create page otherwise page_name = page_data['page'] path = os.path.join('sources', page_name) if os.path.exists(path): template = read_file(path) assert '{{autogenerated}}' in template, ('Template found for ' + path + ' but missing {{autogenerated}} tag.') mkdown = template.replace('{{autogenerated}}', mkdown) print('...inserting autogenerated content into template:', path) else: print('...creating new page with autogenerated content:', path) subdir = os.path.dirname(path) if not os.path.exists(subdir): os.makedirs(subdir) with open(path, 'w') as f: f.write(mkdown) shutil.copyfile('../CONTRIBUTING.md', 'sources/contributing.md')