dev(hansbug): merge from main

.github/workflows/badge.yml

@@ -3,8 +3,6 @@ name: Badge Creation
 on:
   push:
     branches: [ main, 'badge/*', 'doc/*' ]
-  pull_request:
-    branches: [ main, 'badge/*', 'doc/*' ]
 
 jobs:
   update-badges:

.github/workflows/doc.yml

@@ -6,8 +6,6 @@ name: Docs Deploy
 on:
   push:
     branches: [ main, 'doc/*', 'dev/*' ]
-  pull_request:
-    branches: [ main, 'doc/*', 'dev/*' ]
   release:
     types: [ published ]
 

.github/workflows/run.yml

+name: Code Script Run
+
+on:
+  push:
+    branches: [ main, 'dev/*', 'run/*' ]
+
+jobs:
+  unittest:
+    name: Code Script Run
+    runs-on: ${{ matrix.os }}
+    if: "!contains(github.event.head_commit.message, 'ci skip')"
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - 'ubuntu-18.04'
+        python-version:
+          - '3.6'
+          - '3.7'
+          - '3.8'
+          - '3.9'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 20
+      - name: Set up system dependences on linux
+        if: ${{ runner.os == 'Linux' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y tree cloc wget curl make graphviz
+          sudo apt-get install -y libxml2-dev libxslt-dev python-dev  # need by pypy3
+          dot -V
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade flake8 setuptools wheel twine
+          pip install -r requirements.txt
+          pip install -r requirements-build.txt
+          pip install -r requirements-test.txt
+          pip install .
+      - name: Test the basic environment
+        run: |
+          python -V
+          pip --version
+          pip list
+          tree .
+          cloc treevalue
+          cloc test
+      - name: Run Script
+        env:
+          CI: 'true'
+          LINETRACE: 1
+        run: |
+          make clean build run
+      - name: Show the artifacts
+        run: |
+          tree runs/artifacts
+      - uses: actions/upload-artifact@v2
+        with:
+          name: run-artifacts-${{ matrix.os }}-${{ matrix.python-version }}
+          path: runs/artifacts

.github/workflows/test.yml

@@ -2,7 +2,6 @@ name: Code Test
 
 on:
   - push
-  - pull_request
 
 jobs:
   unittest:
@@ -43,6 +42,7 @@ jobs:
           pip install -r requirements.txt
           pip install -r requirements-build.txt
           pip install -r requirements-test.txt
+          ./install_test.sh
       - name: Test the basic environment
         run: |
           python -V
@@ -104,6 +104,7 @@ jobs:
           pip install -r requirements.txt
           pip install -r requirements-build.txt
           pip install -r requirements-test.txt
+          ./install_test.sh
       - name: Test the basic environment
         run: |
           python -V
@@ -153,4 +154,13 @@ jobs:
       - name: Build packages
         run: |
           make package
-          ls -al dist
\ No newline at end of file
+          ls -al dist
+      - name: Get package version
+        run: |
+          make build
+          python -c 'from treevalue.config.meta import __VERSION__;print(__VERSION__)'
+          echo "PACKAGE_VERSION=$(python -c 'from treevalue.config.meta import __VERSION__;print(__VERSION__)')" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v2
+        with:
+          name: treevalue-v${{ env.PACKAGE_VERSION }}-${{ matrix.os }}-${{ matrix.python-version }}
+          path: dist

.gitignore

@@ -1197,6 +1197,7 @@ fabric.properties
 .python-version
 /docs/build
 /public
+/.installs
 /docs/source/**/*.puml.svg
 /docs/source/**/*.puml.png
 /docs/source/**/*.gv.svg
@@ -1208,4 +1209,6 @@ fabric.properties
 /docs/source/**/*.sh.err
 /docs/source/**/*.sh.exitcode
 /docs/source/**/*.dat.*
-!/docs/source/_static/**/*
\ No newline at end of file
+!/docs/source/_static/**/*
+!/runs/artifacts
+/runs/artifacts/*
\ No newline at end of file

Makefile

@@ -7,6 +7,7 @@ DIST_DIR       := ./dist
 WHEELHOUSE_DIR := ./wheelhouse
 TEST_DIR       := ./test
 SRC_DIR        := ./treevalue
+RUNS_DIR       := ./runs
 
 RANGE_DIR      ?= .
 RANGE_TEST_DIR := ${TEST_DIR}/${RANGE_DIR}
@@ -57,4 +58,7 @@ benchmark:
 docs:
 	$(MAKE) -C "${DOC_DIR}" build
 pdocs:
-	$(MAKE) -C "${DOC_DIR}" prod
\ No newline at end of file
+	$(MAKE) -C "${DOC_DIR}" prod
+
+run:
+	PYTHONPATH=$(shell readlink -f .):$(shell readlink -f ${RUNS_DIR}):${PYTHONPATH} $(MAKE) -C "${RUNS_DIR}" run

docs/source/api_doc/tree/general.rst

@@ -7,7 +7,7 @@ FastTreeValue
 -------------------
 
 .. autoclass:: treevalue.tree.general.FastTreeValue
-    :members: _attr_extern, json, clone, __add__, __radd__, __sub__, __rsub__, __mul__, __rmul__, __matmul__, __rmatmul__, __truediv__, __rtruediv__, __floordiv__, __rfloordiv__, __mod__, __rmod__, __pow__, __rpow__, __and__, __rand__, __or__, __ror__, __xor__, __rxor__, __lshift__, __rlshift__, __rshift__, __rrshift__, __pos__, __neg__, __invert__, __getitem__, __setitem__, __delitem__, __call__, __getattribute__, __setattr__, __delattr__, __repr__, __iter__, __hash__, __eq__, map, type, mask, filter, __str__, reduce, rise, union, subside, __getstate__, __setstate__, __iadd__, __isub__, __imul__, __imatmul__, __ifloordiv__, __itruediv__, __ipow__, __imod__, __iand__, __ior__, __ixor__, __ilshift__, __irshift__, graph, graphics, func, walk
+    :members: _attr_extern, json, clone, __add__, __radd__, __sub__, __rsub__, __mul__, __rmul__, __matmul__, __rmatmul__, __truediv__, __rtruediv__, __floordiv__, __rfloordiv__, __mod__, __rmod__, __pow__, __rpow__, __and__, __rand__, __or__, __ror__, __xor__, __rxor__, __lshift__, __rlshift__, __rshift__, __rrshift__, __pos__, __neg__, __invert__, __getitem__, __setitem__, __delitem__, __call__, __getattribute__, __setattr__, __delattr__, __repr__, __iter__, __hash__, __eq__, map, type, mask, filter, __str__, reduce, rise, union, subside, __getstate__, __setstate__, __iadd__, __isub__, __imul__, __imatmul__, __ifloordiv__, __itruediv__, __ipow__, __imod__, __iand__, __ior__, __ixor__, __ilshift__, __irshift__, graph, graphics, func, keys, values, items, walk
 
 
 .. _apidoc_tree_general_generaltreevalue:

docs/source/api_doc/tree/tree.rst

@@ -9,7 +9,7 @@ TreeValue
 ---------------
 
 .. autoclass:: TreeValue
-    :members: __init__, __getattribute__, __setattr__, __delattr__, __contains__, __repr__, __iter__, __hash__, __eq__, _attr_extern, __len__, __bool__, __str__, __getstate__, __setstate__, get
+    :members: __init__, __getattribute__, __setattr__, __delattr__, __contains__, __repr__, __iter__, __hash__, __eq__, _attr_extern, __len__, __bool__, __str__, __getstate__, __setstate__, get, keys, values, items
 
 
 .. _apidoc_tree_tree_delayed:

install_test.sh

+mkdir -p .installs
+
+git clone --depth=1 https://github.com/facebookresearch/torchbeast.git .installs/torchbeast
+cd .installs/torchbeast/nest
+CXX=c++ pip install . -vv
+cd ../../..

runs/Makefile

+.PHONY: all run
+
+all: run
+run:
+	echo "hello world"
+	echo ${PYTHONPATH}
+
+clean:
+	rm -rf artifacts/*
+	touch artifacts/.keep

test/compare/facebook/test_nest.py

+try:
+    import nest
+except ImportError:
+    nest = None
+import pytest
+
+from treevalue import FastTreeValue, flatten, mapping, func_treelize, unflatten, union
+
+_TREE_DATA_1 = {'a': 1, 'b': 2, 'x': {'c': 3, 'd': 4}}
+_TREE_1 = FastTreeValue(_TREE_DATA_1)
+
+_UMARK = pytest.mark.benchmark(group='facebook-nest') if nest is not None else pytest.mark.ignore
+
+
+@_UMARK
+class TestCompareFacebookNest:
+    N = 5
+
+    def __create_nested_tree_data(self, n):
+        return {
+            ('no_%04d' % (i + 1,)): _TREE_DATA_1 for i in range(n)
+        }
+
+    def __create_nested_tree(self, n):
+        return FastTreeValue(self.__create_nested_tree_data(n))
+
+    @pytest.mark.parametrize('n', [2 ** i for i in range(N)])
+    def test_nest_flatten(self, benchmark, n):
+        benchmark(nest.flatten, self.__create_nested_tree_data(n))
+
+    @pytest.mark.parametrize('n', [2 ** i for i in range(N)])
+    def test_tv_flatten(self, benchmark, n):
+        benchmark(flatten, self.__create_nested_tree(n))
+
+    def test_nest_pack_as(self, benchmark):
+        benchmark(nest.pack_as, _TREE_DATA_1, nest.flatten(_TREE_DATA_1))
+
+    def test_tv_unflatten(self, benchmark):
+        benchmark(unflatten, flatten(_TREE_1))
+
+    @pytest.mark.parametrize('n', [2 ** i for i in range(N)])
+    def test_nest_map(self, benchmark, n):
+        benchmark(nest.map, lambda x: x ** 2, self.__create_nested_tree_data(n))
+
+    @pytest.mark.parametrize('n', [2 ** i for i in range(N)])
+    def test_tv_map(self, benchmark, n):
+        benchmark(mapping, self.__create_nested_tree(n), lambda x: x ** 2)
+
+    def test_nest_map_many2(self, benchmark):
+        def f(a, b):
+            return a ** b + a * b
+
+        benchmark(nest.map_many2, f, _TREE_DATA_1, _TREE_DATA_1)
+
+    def test_nest_map_many(self, benchmark):
+        def f(a):
+            return a[0] ** a[1] + a[0] * a[1]
+
+        benchmark(nest.map_many, f, _TREE_DATA_1, _TREE_DATA_1)
+
+    def test_tv_treelize_call(self, benchmark):
+        @func_treelize()
+        def f(a, b):
+            return a ** b + a * b
+
+        benchmark(f, _TREE_1, _TREE_1)
+
+    def test_tv_mapping_union(self, benchmark):
+        def f(a):
+            return a[0] ** a[1]
+
+        def _my_func(fx, *v):
+            return mapping(union(*v), fx)
+
+        benchmark(_my_func, f, _TREE_1, _TREE_1)

test/entry/cli/test_graph.py

@@ -35,9 +35,9 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 5830 <= os.path.getsize('test_graph.svg') <= 5930
+            assert 5000 <= os.path.getsize('test_graph.svg') <= 7000
             assert os.path.exists('test_graph.gv')
-            assert 1800 <= os.path.getsize('test_graph.gv') <= 1900
+            assert 1500 <= os.path.getsize('test_graph.gv') <= 2500
 
     def test_simple_code_graph_to_stdout(self):
         runner = CliRunner()
@@ -52,7 +52,7 @@ class TestEntryCliGraph:
             assert result.exit_code == 0
             assert not os.path.exists('test_graph.svg')
             assert not os.path.exists('test_graph.gv')
-            assert 1800 <= len(result.output) <= 1900
+            assert 1500 <= len(result.output) <= 2500
 
     def test_simple_code_multiple_graph(self):
         runner = CliRunner()
@@ -64,7 +64,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 11000 <= os.path.getsize('test_graph.svg') <= 12000
+            assert 10000 <= os.path.getsize('test_graph.svg') <= 13000
 
         with runner.isolated_filesystem():
             result = runner.invoke(
@@ -74,7 +74,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 16150 <= os.path.getsize('test_graph.svg') <= 16300
+            assert 15500 <= os.path.getsize('test_graph.svg') <= 17500
 
     def test_simple_binary_graph(self):
         runner = CliRunner()
@@ -89,7 +89,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 5830 <= os.path.getsize('test_graph.svg') <= 5930
+            assert 5500 <= os.path.getsize('test_graph.svg') <= 6500
 
         with runner.isolated_filesystem():
             with open('g1.bg', 'wb') as file:
@@ -106,7 +106,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 16150 <= os.path.getsize('test_graph.svg') <= 16300
+            assert 15500 <= os.path.getsize('test_graph.svg') <= 17500
 
         with runner.isolated_filesystem():
             with open('test.entry.cli.test_graph.t1', 'wb') as file:
@@ -119,7 +119,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 11000 <= os.path.getsize('test_graph.svg') <= 12000
+            assert 10000 <= os.path.getsize('test_graph.svg') <= 13000
 
         with runner.isolated_filesystem():
             with open('test.entry.cli.test_graph.t1', 'wb') as file:
@@ -132,7 +132,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 5830 <= os.path.getsize('test_graph.svg') <= 5930
+            assert 5500 <= os.path.getsize('test_graph.svg') <= 6500
 
     def test_duplicates(self):
         runner = CliRunner()
@@ -145,7 +145,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 10000 <= os.path.getsize('test_graph.svg') <= 10430
+            assert 8000 <= os.path.getsize('test_graph.svg') <= 12000
 
         _p = os.path.abspath(os.curdir)
         with runner.isolated_filesystem():
@@ -160,7 +160,7 @@ class TestEntryCliGraph:
             assert os.path.exists('test_graph.svg')
             import shutil
             shutil.copy('test_graph.svg', os.path.join(_p, 'test_graph.svg'))
-            assert 10550 <= os.path.getsize('test_graph.svg') <= 10650
+            assert 10000 <= os.path.getsize('test_graph.svg') <= 11000
 
     def test_graph(self):
         runner = CliRunner()
@@ -174,7 +174,7 @@ class TestEntryCliGraph:
 
             assert result.exit_code == 0
             assert os.path.exists('test_graph.svg')
-            assert 15670 <= os.path.getsize('test_graph.svg') <= 15770
+            assert 14000 <= os.path.getsize('test_graph.svg') <= 16500
 
             content = pathlib.Path('test_graph.svg').read_text()
             assert 'first title' not in content
@@ -190,7 +190,7 @@ class TestEntryCliGraph:
             )
 
             assert result.exit_code == 0
-            assert 5320 <= len(result.output) <= 5420
+            assert 5000 <= len(result.output) <= 6000
             assert '#ffffff00' in result.output
 
         with runner.isolated_filesystem():
@@ -221,4 +221,4 @@ class TestEntryCliGraph:
 
                 assert result.exit_code == 0
                 assert os.path.exists('test_graph.svg')
-                assert 700 <= os.path.getsize('test_graph.svg') <= 800
+                assert 500 <= os.path.getsize('test_graph.svg') <= 1000

test/tree/tree/test_tree.py

@@ -204,3 +204,34 @@ class TestTreeTreeTree:
         with pytest.raises(KeyError):
             _ = tv1.get('e')
         assert tv1.get('e', 233) == 233
+
+    def test_keys(self):
+        tv1 = TreeValue({'a': 1, 'b': 2, 'c': {'x': 2, 'y': 3}, 'd': raw({'x': 2, 'y': 3})})
+        assert set(tv1.keys()) == {'a', 'b', 'c', 'd'}
+
+    def test_values(self):
+        tv1 = TreeValue({'a': 1, 'b': 2, 'c': {'x': 2, 'y': 3}})
+        assert set(tv1.c.values()) == {2, 3}
+        assert len(list(tv1.values())) == 3
+        assert 1 in tv1.values()
+        assert 2 in tv1.values()
+
+    def test_items(self):
+        tv1 = TreeValue({'a': 1, 'b': 2, 'c': {'x': 2, 'y': 3}, 'd': raw({'x': 2, 'y': 3})})
+        assert sorted(tv1.items()) == [
+            ('a', 1),
+            ('b', 2),
+            ('c', TreeValue({'x': 2, 'y': 3})),
+            ('d', {'x': 2, 'y': 3}),
+        ]
+
+        class MyTreeValue(TreeValue):
+            pass
+
+        tv2 = MyTreeValue({'a': 1, 'b': 2, 'c': {'x': 2, 'y': 3}, 'd': raw({'x': 2, 'y': 3})})
+        assert sorted(tv2.items()) == [
+            ('a', 1),
+            ('b', 2),
+            ('c', MyTreeValue({'x': 2, 'y': 3})),
+            ('d', {'x': 2, 'y': 3}),
+        ]

test/utils/test_formattree.py

@@ -34,8 +34,8 @@ from treevalue.utils import (
 @pytest.mark.unittest
 class TestFormatTree(TestCase):
 
-    def format_tree(self, tree):
-        return format_tree(tree, itemgetter(0), itemgetter(1))
+    def format_tree(self, tree, encoding='utf8'):
+        return format_tree(tree, itemgetter(0), itemgetter(1), encoding)
 
     def test_single_node_tree(self):
         tree = ('foo', [])
@@ -60,6 +60,22 @@ class TestFormatTree(TestCase):
         └── qux
         '''), output)
 
+    def test_single_level_tree_with_ascii(self):
+        tree = (
+            'foo', [
+                ('bar', []),
+                ('baz', []),
+                ('qux', []),
+            ],
+        )
+        output = self.format_tree(tree, encoding='ascii')
+        self.assertEqual(dedent(u'''\
+        foo
+        +-- bar
+        +-- baz
+        `-- qux
+        '''), output)
+
     def test_multi_level_tree(self):
         tree = (
             'foo', [

treevalue/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = "treevalue"
 
 #: Version of this project.
-__VERSION__ = "1.2.0"
+__VERSION__ = "1.2.1"
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'A flexible, generalized tree-based data structure.'

treevalue/tree/tree/tree.pyx

@@ -370,6 +370,50 @@ cdef class TreeValue:
         """
         return self._st
 
+    @cython.binding(True)
+    def keys(self):
+        """
+        Overview:
+            Get keys of this treevalue object, like the :class:`dict`.
+
+        Returns:
+            - keys: A generator of all the keys.
+        """
+        return self._st.keys()
+
+    @cython.binding(True)
+    def values(self):
+        """
+        Overview:
+            Get value of this treevalue object, like the :class:`dict`.
+
+        Returns:
+            - values: A generator of all the values
+        """
+        cdef object v
+        for v in self._st.values():
+            if isinstance(v, TreeStorage):
+                yield self._type(v)
+            else:
+                yield v
+
+    @cython.binding(True)
+    def items(self):
+        """
+        Overview:
+            Get pairs of keys and values of this treevalue object, like the :class:`items`.
+
+        Returns:
+            - items: A generator of pairs of keys and values.
+        """
+        cdef str k
+        cdef object v
+        for k, v in self._st.items():
+            if isinstance(v, TreeStorage):
+                yield k, self._type(v)
+            else:
+                yield k, v
+
 cdef str _prefix_fix(object text, object prefix):
     cdef list lines = []
     cdef int i

treevalue/utils/formattree.py

@@ -35,43 +35,45 @@ Changes:
 
 import itertools
 import os
+import sys
 
-FORK = u'\u251c'
-LAST = u'\u2514'
-VERTICAL = u'\u2502'
-HORIZONTAL = u'\u2500'
-NEWLINE = u''
+_DEFAULT_ENCODING = os.environ.get("PYTHONIOENCODING", sys.getdefaultencoding())
 
+_UTF8_CHARS = (u'\u251c', u'\u2514', u'\u2502', u'\u2500', u'')
+_ASCII_CHARS = (u'+', u'`', u'|', u'-', u'')
 
-def _format_newlines(prefix, formatted_node):
+
+def _format_newlines(prefix, formatted_node, chars: tuple):
     """
     Convert newlines into U+23EC characters, followed by an actual newline and
     then a tree prefix so as to position the remaining text under the previous
     line.
     """
+    FORK, LAST, VERTICAL, HORIZONTAL, NEWLINE = chars
     replacement = u''.join([NEWLINE, os.linesep, prefix])
     return replacement.join(formatted_node.splitlines())
 
 
-def _format_tree(node, format_node, get_children, prefix=u''):
+def _format_tree(node, format_node, get_children, prefix=u'', chars: tuple = _UTF8_CHARS):
+    FORK, LAST, VERTICAL, HORIZONTAL, NEWLINE = chars
     children = list(get_children(node))
     next_prefix = u''.join([prefix, VERTICAL, u'   '])
     for child in children[:-1]:
         yield u''.join([
             prefix, FORK, HORIZONTAL, HORIZONTAL, u' ',
-            _format_newlines(next_prefix, format_node(child))])
-        for result in _format_tree(child, format_node, get_children, next_prefix):
+            _format_newlines(next_prefix, format_node(child), chars)])
+        for result in _format_tree(child, format_node, get_children, next_prefix, chars):
             yield result
     if children:
         last_prefix = u''.join([prefix, u'    '])
         yield u''.join([
             prefix, LAST, HORIZONTAL, HORIZONTAL, u' ',
-            _format_newlines(last_prefix, format_node(children[-1]))])
-        for result in _format_tree(children[-1], format_node, get_children, last_prefix):
+            _format_newlines(last_prefix, format_node(children[-1]), chars)])
+        for result in _format_tree(children[-1], format_node, get_children, last_prefix, chars):
             yield result
 
 
-def format_tree(node, format_node, get_children) -> str:
+def format_tree(node, format_node, get_children, encoding=None) -> str:
     r"""
     Overview:
         Format the given tree.
@@ -80,6 +82,8 @@ def format_tree(node, format_node, get_children) -> str:
         - node: Node object
         - format_node: Format node getter
         - get_children: Children getter.
+        - encoding: Encoding to be used. Default is ``None`` which means system encoding. \
+            When ASCII encoding is used, ASCII chars will be used instead of original chars.
 
     Returns:
         - formatted: Formatted string.
@@ -111,9 +115,13 @@ def format_tree(node, format_node, get_children) -> str:
             └── c
                 d
     """
-    lines = itertools.chain(
+    if 'ASCII' in (encoding or _DEFAULT_ENCODING).upper():
+        _chars = _ASCII_CHARS
+    else:
+        _chars = _UTF8_CHARS
+
+    return os.linesep.join(itertools.chain(
         [format_node(node)],
-        _format_tree(node, format_node, get_children),
+        _format_tree(node, format_node, get_children, u'', _chars),
         [u''],
-    )
-    return os.linesep.join(lines)
+    ))