diff --git a/Dockerfile b/Dockerfile index 0f3d8be423130a6e7109e9b97f3c7b07b4bcfe92..f783023ace499e9949a3f01fa740b8bb302712c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,10 +18,6 @@ RUN apt-get update \ iputils-ping \ && rm -rf /var/lib/apt/lists/* -# Install language packs -RUN pip install --upgrade pip \ - && pip install jupyterlab-language-pack-zh-CN - # Install Java # Install dependencies RUN apt-get update \ @@ -72,12 +68,22 @@ RUN pip install --no-cache-dir jupyter-c-kernel/ \ # Cleanup RUN rm -rf jupyter-c-kernel/ +# 3.1.9 <= 3.1.6 +# RUN pip install --upgrade jupyterlab + +# Install language packs +RUN pip install --no-cache-dir pip install jupyterlab-language-pack-zh-CN==0.0.2.dev0 + USER $NB_USER COPY requirements.txt /tmp/requirements.txt RUN python -m pip install --no-cache-dir \ -r /tmp/requirements.txt +COPY nbgitpuller/ /tmp/nbgitpuller/ +RUN python -m pip install --no-cache-dir /tmp/nbgitpuller +# RUN python -m pip install --no-cache-dir git+https://codechina.csdn.net/codechina_dev/nbgitpuller.git@develop + # Support overriding a package or two through passed docker --build-args. # ARG PIP_OVERRIDES="jupyterhub==1.3.0" ARG PIP_OVERRIDES= @@ -89,5 +95,6 @@ RUN jupyter serverextension enable --py nbgitpuller --sys-prefix # Uncomment the line below to make nbgitpuller default to start up in JupyterLab ENV NBGITPULLER_APP=lab +ENV NBGITPULLER_PARENTPATH=tmp # conda/pip/apt install additional packages here, if desired. diff --git a/jupyterlab_language_pack_zh_CN-0.0.1.dev0-py2.py3-none-any.whl b/jupyterlab_language_pack_zh_CN-0.0.1.dev0-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..6b9479c263302b20bebe986e30b739c6f517162c --- /dev/null +++ b/jupyterlab_language_pack_zh_CN-0.0.1.dev0-py2.py3-none-any.whl @@ -0,0 +1,846 @@ + + + + + + + + + + + + + + + + + + + + + + +jupyterlab_language_pack_zh_CN-0.0.1.dev0-py2.py3-none-any.whl · master · Miykael_xxm / Jupyter 101 · CODE CHINA + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Skip to content +
+
+
+

+

+ + +
+
+ +
+ +
+
+
+ + + +
+ + +
+
+
+
+
J
+
+
+Jupyter 101 +
+
+ +
+
+
+
+
+
+
+

+Miykael_xxm / Jupyter 101 + +

+
+ +
+
+
+
+
+
+ +通知 + +4 + +
+
+ +
+
+
+ +Star + +1 + +
+ +
+ +Fork + +0 + +
+ +
+
+
+
+ + +
+ +
+ +
+
+
+
+
J
+
+
+Jupyter 101 +
+
+ +
+
+ + +
+
+ +
+ + + +
+
+ +
+
+

VS Code《CSDN工作流》上新啦！

+
+
+ +
+
+ + + + + + + + + + +
+
+
+
+
+
+ +
+ +
+
+
+
+ +
+ +
+ +
+
+
切换分支/标签
+
+
+
+
+
+
+
+
+ +
+
查找文件 +历史永久链接Permalink
+
+ +
+
+
    +
  • +
    +Miykael_xxm's avatar +
    +
    +
    +update + +· +8c49a832 + +
    +由 Miykael_xxm 提交于 +
    + +
    +
    + +
    +
    +8c49a832 +
    + + +
    +
    +
    +
    • + +
+
+ + +
+
+ +
+ + + +
+ + +
+
+
+
+ + + + + + + + + diff --git a/nbgitpuller/.circleci/config.yml b/nbgitpuller/.circleci/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..e12c7e5b4ecada159b155e80f42f8d6a7d9db281 --- /dev/null +++ b/nbgitpuller/.circleci/config.yml @@ -0,0 +1,65 @@ +version: 2.1 +jobs: + build_docs: + docker: + - image: circleci/python:3.6-stretch + steps: + - build_site + - store_artifacts: + path: docs/_build/html/ + destination: html + + push_docs: + docker: + - image: circleci/python:3.6-stretch + steps: + # Add deployment key fingerprint for CircleCI to use for a push + - add_ssh_keys: + fingerprints: + # The SSH key fingerprint + - "c5:70:b9:1b:9a:cf:e3:88:25:9f:33:8e:ee:09:76:9f" + + - build_site + + - run: + name: Pushing documentation to gh-pages + command: | + pip install --user ghp-import + ghp-import --no-jekyll --push --message "Update documentation [skip ci]" docs/_build/html + +workflows: + version: 2 + default: + jobs: + - build_docs + - push_docs: + filters: # using regex filters requires the entire branch to match + branches: + only: # only branches matching the below regex filters will run + - main + +commands: + build_site: + description: "Build the site with sphinx" + steps: + # Get our data and merge with upstream + - run: sudo apt-get update + - checkout + # Python env + - run: echo "export PATH=~/.local/bin:$PATH" >> $BASH_ENV + + - restore_cache: + keys: + - cache-pip + - run: pip install --user -r docs/doc-requirements.txt + - save_cache: + key: cache-pip + paths: + - ~/.cache/pip + + # Build the docs + - run: + name: Build docs to store + command: | + cd docs + make html diff --git a/nbgitpuller/.flake8 b/nbgitpuller/.flake8 new file mode 100644 index 0000000000000000000000000000000000000000..5619889bc961a8bf79b38255013f0209c99e9d37 --- /dev/null +++ b/nbgitpuller/.flake8 @@ -0,0 +1,12 @@ +[flake8] +# Ignore style and complexity +# E: style errors +# W: style warnings +# C: complexity +# E402: module level import not at top of file +# I100: Import statements are in the wrong order +# I101: Imported names are in the wrong order. Should be +ignore = E, C, W, E402, I100, I101, D400 +exclude = + .cache, + .github diff --git a/nbgitpuller/.github/workflows/publish.yml b/nbgitpuller/.github/workflows/publish.yml new file mode 100644 index 0000000000000000000000000000000000000000..3dfe659f31093ac6b6eaf616974b6eb14fdf5be3 --- /dev/null +++ b/nbgitpuller/.github/workflows/publish.yml @@ -0,0 +1,35 @@ +# Build releases and (on tags) publish to PyPI +name: Release + +# always build releases (to make sure wheel-building works) +# but only publish to PyPI on tags +on: + push: + pull_request: + +jobs: + build-release: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: install build package + run: | + pip install --upgrade pip + pip install build + pip freeze + + - name: build release + run: | + python -m build --sdist --wheel . + ls -l dist + + - name: publish to pypi + uses: pypa/gh-action-pypi-publish@v1.4.1 + if: startsWith(github.ref, 'refs/tags/') + with: + user: __token__ + password: ${{ secrets.pypi_password }} diff --git a/nbgitpuller/.github/workflows/test.yml b/nbgitpuller/.github/workflows/test.yml new file mode 100644 index 0000000000000000000000000000000000000000..89168d6186d2ac1a4fd1fb0ae5363097c1703622 --- /dev/null +++ b/nbgitpuller/.github/workflows/test.yml @@ -0,0 +1,54 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions +# +name: Tests + +on: + pull_request: + push: + workflow_dispatch: + +jobs: + test: + + runs-on: ubuntu-20.04 + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v2 + + - name: Run webpack to build static assets + run: | + npm install + npm run webpack + + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + # DISABLED: Since we don't pin our dependencies in dev-requirements.txt + # and only refresh the cache when it changes, we end up with a + # cache that remains for too long and cause failures. Due to + # this, it has been disabled. + # + # - name: Cache pip dependencies + # uses: actions/cache@v2 + # with: + # path: ~/.cache/pip + # # Look to see if there is a cache hit for the corresponding requirements file + # key: ${{ runner.os }}-pip-${{ hashFiles('*requirements.txt') }} + # restore-keys: | + # ${{ runner.os }}-pip- + + - name: Install dependencies + run: | + pip install -r dev-requirements.txt + + - name: Run flake8 linter + run: flake8 + + - name: Run tests + run: | + pip install . + pytest --verbose --maxfail=2 --color=yes --cov nbgitpuller diff --git a/nbgitpuller/.gitignore b/nbgitpuller/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2d5462d17dbff0a79e42c299704250946ab36a57 --- /dev/null +++ b/nbgitpuller/.gitignore @@ -0,0 +1,24 @@ +*.pyc + +dist/ +build/ +*.egg-info/ + +.tox/ +.coverage + +.DS_Store +.cache/ +data8assets/ +.autopull_list +summer/ +test-repo/ +venv/ + +.ipynb_checkpoints +docs/_build + +node_modules/ +package-lock.json + +nbgitpuller/static/dist \ No newline at end of file diff --git a/nbgitpuller/CHANGELOG.md b/nbgitpuller/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..d93bbcda389a978c69fbaaf2905bb930f75755ed --- /dev/null +++ b/nbgitpuller/CHANGELOG.md @@ -0,0 +1,138 @@ +## 0.10 + +### 0.10.2 - 2021-08-25 + +This is a critical security release, please upgrade to this and see [GHSA-mq5p-2mcr-m52j](https://github.com/jupyterhub/nbgitpuller/security/advisories/GHSA-mq5p-2mcr-m52j) more information. + +### 0.10.1 - 2021-06-24 + +#### Bugs fixed + +- Added branch name back to command-line usage [#185](https://github.com/jupyterhub/nbgitpuller/pull/185) ([@sean-morris](https://github.com/sean-morris)) + +#### Documentation improvements + +- Provide cleaner feedback for lint vs test failures [#181](https://github.com/jupyterhub/nbgitpuller/pull/181) ([@yuvipanda](https://github.com/yuvipanda)) + +#### Continuous integration + +- Fix CI failures by disabling pip cache [#188](https://github.com/jupyterhub/nbgitpuller/pull/188) ([@consideRatio](https://github.com/consideRatio)) + +#### Contributors to this release + +([GitHub contributors page for this release](https://github.com/jupyterhub/nbgitpuller/graphs/contributors?from=2021-06-09&to=2021-06-24&type=c)) + +[@consideRatio](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3AconsideRatio+updated%3A2021-06-09..2021-06-24&type=Issues) | [@manics](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Amanics+updated%3A2021-06-09..2021-06-24&type=Issues) | [@sean-morris](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Asean-morris+updated%3A2021-06-09..2021-06-24&type=Issues) | [@yuvipanda](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Ayuvipanda+updated%3A2021-06-09..2021-06-24&type=Issues) + +### 0.10.0 - 2021-06-09 + +#### Enhancements made + +- UI: Branch input placeholder no longer suggests master branch [#180](https://github.com/jupyterhub/nbgitpuller/pull/180) ([@sean-morris](https://github.com/sean-morris)) +- Automatically detect default branch name [#179](https://github.com/jupyterhub/nbgitpuller/pull/179) ([@sean-morris](https://github.com/sean-morris)) +- Tell users about `main` vs `master` branches [#170](https://github.com/jupyterhub/nbgitpuller/pull/170) ([@yuvipanda](https://github.com/yuvipanda)) +- Support generating shiny links [#165](https://github.com/jupyterhub/nbgitpuller/pull/165) ([@yuvipanda](https://github.com/yuvipanda)) + +#### Bugs fixed + +- Handle lack of trailing slashes in hub URLs [#173](https://github.com/jupyterhub/nbgitpuller/pull/173) ([@yuvipanda](https://github.com/yuvipanda)) +- Respect path component of JupyterHub url [#172](https://github.com/jupyterhub/nbgitpuller/pull/172) ([@yuvipanda](https://github.com/yuvipanda)) +- Parse ssh git URLs properly [#163](https://github.com/jupyterhub/nbgitpuller/pull/163) ([@yuvipanda](https://github.com/yuvipanda)) +- Fix failure to restore deleted files (use raw output of git ls-files to avoid quoting unicode) [#156](https://github.com/jupyterhub/nbgitpuller/pull/156) ([@manics](https://github.com/manics)) +- Compare current branch to target - don't assume already on target branch locally [#141](https://github.com/jupyterhub/nbgitpuller/pull/141) ([@danlester](https://github.com/danlester)) + +#### Documentation improvements + +- Document restarting notebook process to see changes [#178](https://github.com/jupyterhub/nbgitpuller/pull/178) ([@yuvipanda](https://github.com/yuvipanda)) +- docs: update README.md badges [#175](https://github.com/jupyterhub/nbgitpuller/pull/175) ([@consideRatio](https://github.com/consideRatio)) +- Add best practices recommendation documentation [#169](https://github.com/jupyterhub/nbgitpuller/pull/169) ([@yuvipanda](https://github.com/yuvipanda)) +- Document how to do local development [#162](https://github.com/jupyterhub/nbgitpuller/pull/162) ([@yuvipanda](https://github.com/yuvipanda)) +- Add badges to README.md [#150](https://github.com/jupyterhub/nbgitpuller/pull/150) ([@consideRatio](https://github.com/consideRatio)) + +#### Continuous Integration + +- CI: Replace Travis with GitHub workflow [#161](https://github.com/jupyterhub/nbgitpuller/pull/161) ([@manics](https://github.com/manics)) +- CI: stop triggering CircleCI on automated pushes to gh-pages [#151](https://github.com/jupyterhub/nbgitpuller/pull/151) ([@consideRatio](https://github.com/consideRatio)) + +#### Contributors to this release + +([GitHub contributors page for this release](https://github.com/jupyterhub/nbgitpuller/graphs/contributors?from=2020-08-01&to=2021-06-09&type=c)) + +[@albertmichaelj](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Aalbertmichaelj+updated%3A2020-08-01..2021-06-09&type=Issues) | [@choldgraf](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Acholdgraf+updated%3A2020-08-01..2021-06-09&type=Issues) | [@consideRatio](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3AconsideRatio+updated%3A2020-08-01..2021-06-09&type=Issues) | [@danlester](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Adanlester+updated%3A2020-08-01..2021-06-09&type=Issues) | [@giumas](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Agiumas+updated%3A2020-08-01..2021-06-09&type=Issues) | [@manics](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Amanics+updated%3A2020-08-01..2021-06-09&type=Issues) | [@minrk](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Aminrk+updated%3A2020-08-01..2021-06-09&type=Issues) | [@ryanlovett](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Aryanlovett+updated%3A2020-08-01..2021-06-09&type=Issues) | [@SaladRaider](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3ASaladRaider+updated%3A2020-08-01..2021-06-09&type=Issues) | [@samuelmanzer](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Asamuelmanzer+updated%3A2020-08-01..2021-06-09&type=Issues) | [@sean-morris](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Asean-morris+updated%3A2020-08-01..2021-06-09&type=Issues) | [@ttimbers](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Attimbers+updated%3A2020-08-01..2021-06-09&type=Issues) | [@welcome](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Awelcome+updated%3A2020-08-01..2021-06-09&type=Issues) | [@yuvipanda](https://github.com/search?q=repo%3Ajupyterhub%2Fnbgitpuller+involves%3Ayuvipanda+updated%3A2020-08-01..2021-06-09&type=Issues) + +## 0.9 + +### 0.9.0 - 2020-09-1 + +- Allow destination to be configured ([#42](https://github.com/jupyterhub/nbgitpuller/pull/42)) +- Made the checkout from the reset_deleted_files to use the origin. ([#111](https://github.com/jupyterhub/nbgitpuller/pull/111)) +- Update version. ([#112](https://github.com/jupyterhub/nbgitpuller/pull/112)) +- Update index.rst ([#113](https://github.com/jupyterhub/nbgitpuller/pull/113)) +- Use shallow clones by default ([#117](https://github.com/jupyterhub/nbgitpuller/pull/117)) +- updating theme ([#126](https://github.com/jupyterhub/nbgitpuller/pull/126)) +- Update ipynb with newer query parameters and toggles ([#127](https://github.com/jupyterhub/nbgitpuller/pull/127)) +- Add a mybinder.org tab to the link builder ([#129](https://github.com/jupyterhub/nbgitpuller/pull/129)) +- tab activation on link generator ([#132](https://github.com/jupyterhub/nbgitpuller/pull/132)) +- fixing bug ([#134](https://github.com/jupyterhub/nbgitpuller/pull/134)) +- Fix typo from ipynb link generator external tool reference ([#136](https://github.com/jupyterhub/nbgitpuller/pull/136)) +- Use the correct branch for contentRepo ([#138](https://github.com/jupyterhub/nbgitpuller/pull/138)) +- Fix file paths or application paths ([#140](https://github.com/jupyterhub/nbgitpuller/pull/140)) +- Make the environment repo branch required for binder ([#143](https://github.com/jupyterhub/nbgitpuller/pull/143)) +- Travis pypi deployment, README fixes ([#145](https://github.com/jupyterhub/nbgitpuller/pull/145)) +- Replace data-8 with jupyterhub ([#146](https://github.com/jupyterhub/nbgitpuller/pull/146)) +- CI: fix broken test assertions following --depth 1 by default ([#147](https://github.com/jupyterhub/nbgitpuller/pull/147)) +- CI: ensure tox run's flake8 as well ([#148](https://github.com/jupyterhub/nbgitpuller/pull/148)) + +## 0.8 + +### 0.8.0 2019-11-23 + +- Link generator: init application type from query params ([#107](https://github.com/jupyterhub/nbgitpuller/pull/107)) +- Made the checkout from the reset_deleted_files to use the origin. ([#111](https://github.com/jupyterhub/nbgitpuller/pull/111)) + +## 0.7 + +### 0.7.2 - 2019-10-3 + +- Bump version number ([#103](https://github.com/jupyterhub/nbgitpuller/pull/103)) +- Set authorship info on each commit, rather than repo-wide ([#104](https://github.com/jupyterhub/nbgitpuller/pull/104)) +- Bump version number ([#105](https://github.com/jupyterhub/nbgitpuller/pull/105)) + +### 0.7.1 2019-10-3 + +- Update version to 0.7.0. ([#100](https://github.com/jupyterhub/nbgitpuller/pull/100)) +- Fix legacy links with empty path ([#102](https://github.com/jupyterhub/nbgitpuller/pull/102)) +- Bump version number ([#103](https://github.com/jupyterhub/nbgitpuller/pull/103)) + +### 0.7.0 2019-07-31 + +- adding a link generator binder ([#49](https://github.com/jupyterhub/nbgitpuller/pull/49)) +- Clean up link_generator notebook / app ([#50](https://github.com/jupyterhub/nbgitpuller/pull/50)) +- add link to TLJH guide in readme ([#52](https://github.com/jupyterhub/nbgitpuller/pull/52)) +- updating link sanitizing ([#54](https://github.com/jupyterhub/nbgitpuller/pull/54)) +- adds link to a basic video instruction ([#56](https://github.com/jupyterhub/nbgitpuller/pull/56)) +- Add new link generator instructions ([#62](https://github.com/jupyterhub/nbgitpuller/pull/62)) +- adding new nbgitpuller link gen app ([#63](https://github.com/jupyterhub/nbgitpuller/pull/63)) +- Implement depth/shallow-clone support ([#67](https://github.com/jupyterhub/nbgitpuller/pull/67)) +- Made repo_dir an absolute path based on the server_root_dir. ([#71](https://github.com/jupyterhub/nbgitpuller/pull/71)) +- Serve gh pages from docs/ not gh-pages ([#73](https://github.com/jupyterhub/nbgitpuller/pull/73)) +- Pass nbapp along to GitPuller so it can read from our configuration ([#75](https://github.com/jupyterhub/nbgitpuller/pull/75)) +- Rework nbgitpuller link generator ([#76](https://github.com/jupyterhub/nbgitpuller/pull/76)) +- Generate URLs that can be launched from canvas ([#78](https://github.com/jupyterhub/nbgitpuller/pull/78)) +- Don't require including cloned dir name in path to open ([#79](https://github.com/jupyterhub/nbgitpuller/pull/79)) +- adding documentation ([#81](https://github.com/jupyterhub/nbgitpuller/pull/81)) +- circle config to push docs ([#82](https://github.com/jupyterhub/nbgitpuller/pull/82)) +- documentation clarification ([#88](https://github.com/jupyterhub/nbgitpuller/pull/88)) +- Redo documentation ([#92](https://github.com/jupyterhub/nbgitpuller/pull/92)) +- Allow git@example.com:repo links ([#97](https://github.com/jupyterhub/nbgitpuller/pull/97)) + +## 0.6 + +### 0.6.1 2018-07-19 + +- Install Jupyter notebook extension by default, Add missing nbgitpuller.json file + +### 0.6.0 2018-07-18 + +- Work with (and require) newer notebook version ([#46](https://github.com/jupyterhub/nbgitpuller/pull/46)) +- Update README.md ([#48](https://github.com/jupyterhub/nbgitpuller/pull/48)) diff --git a/nbgitpuller/LICENSE b/nbgitpuller/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..c7c4ca7d4a41278d33bf19a3f0869cfb162630d0 --- /dev/null +++ b/nbgitpuller/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2017, YuviPanda, Peter Veerman +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/nbgitpuller/MANIFEST.in b/nbgitpuller/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..607df237649251e00b8fe332f2d4648e7cea5378 --- /dev/null +++ b/nbgitpuller/MANIFEST.in @@ -0,0 +1,5 @@ +include *.md +include LICENSE +include setup.cfg +recursive-include nbgitpuller/static * +recursive-include nbgitpuller/templates * diff --git a/nbgitpuller/README.md b/nbgitpuller/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2fa08a4b30e577b0ba891c5fe61bf7a58d069c94 --- /dev/null +++ b/nbgitpuller/README.md @@ -0,0 +1,28 @@ +# [nbgitpuller](https://github.com/jupyterhub/nbgitpuller) + + +[![GitHub Workflow Status - Test](https://img.shields.io/github/workflow/status/jupyterhub/nbgitpuller/Tests?logo=github&label=tests)](https://github.com/jupyterhub/nbgitpuller/actions) +[![CircleCI build status](https://img.shields.io/circleci/build/github/jupyterhub/nbgitpuller?logo=circleci&label=docs)](https://circleci.com/gh/jupyterhub/nbgitpuller) +[![](https://img.shields.io/pypi/v/nbgitpuller.svg?logo=pypi)](https://pypi.python.org/pypi/nbgitpuller) +[![GitHub](https://img.shields.io/badge/issue_tracking-github-blue?logo=github)](https://github.com/jupyterhub/nbgitpuller/issues) +[![Discourse](https://img.shields.io/badge/help_forum-discourse-blue?logo=discourse)](https://discourse.jupyter.org/c/jupyterhub) +[![Gitter](https://img.shields.io/badge/social_chat-gitter-blue?logo=gitter)](https://gitter.im/jupyterhub/jupyterhub) + +`nbgitpuller` lets you distribute content in a git repository to your students +by having them click a simple link. [Automatic +merging](https://jupyterhub.github.io/nbgitpuller/topic/automatic-merging.html) +ensures that your students are never exposed to `git` directly. It is primarily +used with a JupyterHub, but can also work on students' local computers. + +See [the documentation](https://jupyterhub.github.io/nbgitpuller) for more +information. + +## Installation + +```shell +pip install nbgitpuller +``` + +## Example + +![](https://raw.githubusercontent.com/jupyterhub/nbgitpuller/v0.8.0/docs/_static/nbpuller.gif) diff --git a/nbgitpuller/RELEASE.md b/nbgitpuller/RELEASE.md new file mode 100644 index 0000000000000000000000000000000000000000..981a3c56edd2ca46f66f91d8dfed69a2454d93b9 --- /dev/null +++ b/nbgitpuller/RELEASE.md @@ -0,0 +1,67 @@ +# How to make a release + +`nbgitpuller` is a package available on +[PyPI](https://pypi.org/project/nbgitpuller/) and +[conda-forge](https://anaconda.org/conda-forge/nbgitpuller). +These are instructions on how to make a release on PyPI. +The PyPI release is done automatically by TravisCI when a tag is pushed. + + +## Steps to make a release + +1. Checkout main and make sure it is up to date. + + ```shell + ORIGIN=${ORIGIN:-origin} # set to the canonical remote, e.g. 'upstream' if 'origin' is not the official repo + git checkout main + git fetch $ORIGIN main + git reset --hard $ORIGIN/main + # WARNING! This next command deletes any untracked files in the repo + git clean -xfd + ``` + +1. Set the `__version__` variable in + [`nbgitpuller/version.py`](nbgitpuller/version.py) + and make a commit. + + ``` + git add nbgitpuller/version.py + VERSION=... # e.g. 1.2.3 + git commit -m "release $VERSION" + ``` + +1. Reset the `__version__` variable in + [`nbgitpuller/version.py`](nbgitpuller/version.py) + to an incremented patch version with a `dev` element, then make a commit. + ``` + git add nbgitpuller/version.py + git commit -m "back to dev" + ``` + +1. Push your two commits to main. + + ```shell + # first push commits without a tags to ensure the + # commits comes through, because a tag can otherwise + # be pushed all alone without company of rejected + # commits, and we want have our tagged release coupled + # with a specific commit in main + git push $ORIGIN main + ``` + +1. Create a git tag for the pushed release commit and push it. + + ```shell + git tag -a $VERSION -m $VERSION HEAD~1 + + # then verify you tagged the right commit + git log + + # then push it + git push $ORIGIN refs/tags/$VERSION + ``` + +1. Following the release to PyPI, an automated PR should arrive to + [conda-forge/nbgitpuller-feedstock](https://github.com/conda-forge/nbgitpuller-feedstock), + check for the tests to succeed on this PR and then merge it to successfully + update the package for `conda` on the `conda-forge` channel. diff --git a/nbgitpuller/_config.yml b/nbgitpuller/_config.yml new file mode 100644 index 0000000000000000000000000000000000000000..c7418817439b2f071c93a4a6cee831e996123c0b --- /dev/null +++ b/nbgitpuller/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-slate \ No newline at end of file diff --git a/nbgitpuller/binder/link_generator.ipynb b/nbgitpuller/binder/link_generator.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..fea897ef7d93e6aa4ea5d3f13a4a42f5674e88a4 --- /dev/null +++ b/nbgitpuller/binder/link_generator.ipynb @@ -0,0 +1,164 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generate `nbgitpuller` links for your JupyterHub\n", + "\n", + "When users click an `nbgitpuller` link pointing to your JupyterHub,\n", + "\n", + "1. They are asked to log in to the JupyterHub if they have not already\n", + "2. The git repository referred to in the nbgitpuller link is made up to date in their home directory (keeping local changes if there are merge conflicts)\n", + "3. They are shown the specific notebook / directory referred to in the nbgitpuller link.\n", + "\n", + "This is a great way to distribute materials to students.\n", + "\n", + "# Generate `nbgitpuller` links for your JupyterHub\n", + "\n", + "## Sequence of events when users click an `nbgitpuller` link pointing to your JupyterHub,\n", + "\n", + "1. They are asked to log in to the JupyterHub if they have not already\n", + "2. The git repository referred to in the nbgitpuller link is made up to date in their home directory (keeping local changes if there are merge conflicts)\n", + "3. They are shown the specific notebook / directory referred to in the nbgitpuller link.\n", + "\n", + "This is a great way to distribute materials to students.\n", + "\n", + "## Canvas LMS: Assignment Links vs Custom Fields\n", + "\n", + "The Canvas LMS expects the assignment link to include URL encoded parameters since the request is sent to the External Tool as a POST request (in this case JupyterHub is the External Tool). However, all characters (even those considered safe) after the domain and `next=` part should be URL encoded, such as the `/`, `&`, and `=` characters.\n", + "\n", + "The `Custom Fields` text box in the App -> Settings section, on the other hand, does not expect all characters to be URL encoded. The `/` characters that are assigned as part of the query parameter values should be encoded, but not the `&` and `=` characters.\n", + "\n", + "## Usage\n", + "\n", + "- **Assignment Link**: creates a string value which represents an `Assignment` link by toggling the check box next to the `is_assignment_link` label. If unchecked, the tool will create a string to add to the Custom Field section.\n", + "- **Jupyter Lab Link**: creates a string value which redirects the user to a `Jupyter Lab` workspace instead of the `Jupyter Classic` workspace.\n", + "- **LTI Launches**: adds the route associated to the LTI 1.1 login handler. If disabled, it is assumed that the user is using the default authentication class bound to the root of the `domain_url` value.\n", + "- **Default Values**: to avoid having to enter the same values in the widget's text fields on a repetitive basis, add the string values to the function's parameters. For example, the `branch` parameter defaults to `master`." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "'https://my.hub.com/hub/lti/launch?next=%2Fuser-redirect%2Fgit-pull?repo%3D%26branch%3Dmaster%26urlpath%3Dlab%252Ftree%252F.%252F%253Fautodecode'" + }, + "metadata": {} + } + ], + "source": [ + "import os\n", + "from ipywidgets import interact\n", + "from urllib.parse import urlunparse, urlparse, urlencode, parse_qs, parse_qsl, quote\n", + "from IPython.display import Markdown\n", + "\n", + "\n", + "@interact\n", + "def make_launch_link(is_assignment_link=True, is_jupyterlab=True, is_lti11=True, branch='master', hub_url='https://my.hub.com', repo_url='', urlpath=''):\n", + " \"\"\"\n", + " Generate a launch request which clones and merges source files from a git-based\n", + " repository.\n", + "\n", + " Args:\n", + " is_assignment_link (bool): set to True to create a full assignment link, defaults to True.\n", + " is_jupyterlab (bool): set to True to launch Jupyter Lab workspaces, defaults to True.\n", + " is_lti11 (bool): set to True to initiate launch requests with the LTI 1.1 standard.\n", + " branch (str): git repo branch\n", + " hub_url (str): full hub url which needs to include scheme (http or https) and netloc (full domain).\n", + " repo_url (str): full git repo url which needs to include scheme (http or https), netloc (full domain) and path.\n", + " url_path (str): a path to redirect users to after the workspace has successfully spawned (started).\n", + "\n", + " Returns:\n", + " An interactive IPython.display.Markdown object.\n", + " \"\"\"\n", + "\n", + " # Parse the query to its constituent parts\n", + " domain_scheme, domain_netloc, domain_path, domain_params, domain_query_str, domain_fragment = urlparse(hub_url.strip())\n", + " \n", + " repo_scheme, repo_netloc, repo_path, repo_params, repo_query_str, repo_fragment = urlparse(repo_url.strip())\n", + " folder_from_repo_url_path = os.path.basename(os.path.normpath(repo_path))\n", + " \n", + " # Make sure the path doesn't contain multiple slashes\n", + " if not domain_path.endswith('/'):\n", + " domain_path += '/'\n", + " domain_path += 'user-redirect/git-pull'\n", + " \n", + " # With Canvas using LTI 11 Assignment launch requests all characters after the netloc are considered unsafe.\n", + " # When adding custom parameters within the App Settings -> Custom Fields section, only items after the \n", + " path_encoded = ''\n", + " if is_assignment_link:\n", + " path_encoded = quote(domain_path, safe='')\n", + " else:\n", + " path_encoded = quote(domain_path)\n", + "\n", + " path_redirect_url = f'next={path_encoded}'\n", + " if is_lti11:\n", + " assignment_link_path = f'/hub/lti/launch?next={path_encoded}'\n", + " else:\n", + " assignment_link_path = f'/hub?next={path_encoded}'\n", + " \n", + " # Create a tuple of query params from original domain link\n", + " query_params_from_hub_url = parse_qsl(domain_query_str, keep_blank_values=True)\n", + " \n", + " # Set path based on whether or not the user would like to spawn JupyterLab or Jupyter Classic\n", + " urlpath_workspace = ''\n", + " if is_jupyterlab:\n", + " urlpath_workspace = f'lab/tree/{folder_from_repo_url_path}/{urlpath}?autodecode'\n", + " else:\n", + " urlpath_workspace = f'tree/{folder_from_repo_url_path}/{urlpath}'\n", + " \n", + " # Create a tuple of query params for git functionality. Check whether or not we want to launch with\n", + " # jupyterlab to add additional items to the path.\n", + " query_params_for_git = [('repo', repo_url), ('branch', branch), ('urlpath', urlpath_workspace)]\n", + " \n", + " # Merge query params into one list of tuples\n", + " query_params_all = query_params_from_hub_url + query_params_for_git\n", + " \n", + " # First build urlencoded query params where the &, =, and / are considered safe. Then, percent encode\n", + " # all characters.\n", + " encoded_query_params = urlencode(query_params_all)\n", + " encoded_query_params_without_safe_chars = quote(urlencode(query_params_all), safe='')\n", + " \n", + " assignment_link_url = urlunparse((domain_scheme, domain_netloc, assignment_link_path, domain_params, encoded_query_params_without_safe_chars, domain_fragment))\n", + " path_url = urlunparse(('', '', path_redirect_url, domain_params, encoded_query_params, domain_fragment))\n", + " \n", + " if is_assignment_link:\n", + " return assignment_link_url\n", + " return path_url" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/nbgitpuller/binder/postBuild b/nbgitpuller/binder/postBuild new file mode 100644 index 0000000000000000000000000000000000000000..829840871c5be8e3373dfbda6d6597b9b9074417 --- /dev/null +++ b/nbgitpuller/binder/postBuild @@ -0,0 +1,3 @@ +#!/bin/bash +jupyter nbextension enable --py --sys-prefix appmode +jupyter serverextension enable --py --sys-prefix appmode \ No newline at end of file diff --git a/nbgitpuller/binder/requirements.txt b/nbgitpuller/binder/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..87cc563892a6b234540d41b4f1a0c099e7d82077 --- /dev/null +++ b/nbgitpuller/binder/requirements.txt @@ -0,0 +1,2 @@ +ipywidgets +appmode \ No newline at end of file diff --git a/nbgitpuller/dev-requirements.txt b/nbgitpuller/dev-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..37b15b922cc26cee573f13cf62f31bdd8f7351e4 --- /dev/null +++ b/nbgitpuller/dev-requirements.txt @@ -0,0 +1,6 @@ +six +pytest +pytest-cov +flake8 +nbclassic +nest-asyncio diff --git a/nbgitpuller/docs/Makefile b/nbgitpuller/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9764015b498c764e5b11b7b40b4b401925ed2e53 --- /dev/null +++ b/nbgitpuller/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = Binder +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/nbgitpuller/docs/_static/link_gen/link.js b/nbgitpuller/docs/_static/link_gen/link.js new file mode 100644 index 0000000000000000000000000000000000000000..dd371c81c9ec1afcf07a9940caa98c059d7adaa5 --- /dev/null +++ b/nbgitpuller/docs/_static/link_gen/link.js @@ -0,0 +1,276 @@ +// Pure function that generates an nbgitpuller URL +function generateRegularUrl(hubUrl, urlPath, repoUrl, branch) { + + // assume hubUrl is a valid URL + var url = new URL(hubUrl); + + url.searchParams.set('repo', repoUrl); + + if (urlPath) { + url.searchParams.set('urlpath', urlPath); + } + + if (branch) { + url.searchParams.set('branch', branch); + } + + if (!url.pathname.endsWith('/')) { + url.pathname += '/' + } + url.pathname += 'hub/user-redirect/git-pull'; + + return url.toString(); +} + +function generateCanvasUrl(hubUrl, urlPath, repoUrl, branch) { + // assume hubUrl is a valid URL + var url = new URL(hubUrl); + + var nextUrlParams = new URLSearchParams(); + + nextUrlParams.append('repo', repoUrl); + + if (urlPath) { + nextUrlParams.append('urlpath', urlPath); + } + + if (branch) { + nextUrlParams.append('branch', branch); + } + + var nextUrl = '/hub/user-redirect/git-pull?' + nextUrlParams.toString(); + + if (!url.pathname.endsWith('/')) { + url.pathname += '/' + } + url.pathname += 'hub/lti/launch' + url.searchParams.append('next', nextUrl); + + return url.toString(); +} + +function generateBinderUrl(hubUrl, userName, repoName, branch, urlPath, + contentRepoUrl, contentRepoBranch) { + + var url = new URL(hubUrl); + + var nextUrlParams = new URLSearchParams(); + + nextUrlParams.append('repo', contentRepoUrl); + + if (urlPath) { + nextUrlParams.append('urlpath', urlPath); + } + + if (contentRepoBranch) { + nextUrlParams.append('branch', contentRepoBranch); + } + + var nextUrl = 'git-pull?' + nextUrlParams.toString(); + + var path = '/v2/gh/'; + url.pathname = path.concat(userName, "/", repoName, "/", branch); + url.searchParams.append('urlpath', nextUrl); + + return url.toString(); +} + +var apps = { + classic: { + title: 'Classic Notebook', + generateUrlPath: function (path) { return 'tree/' + path; }, + }, + jupyterlab: { + title: 'JupyterLab', + generateUrlPath: function (path) { return 'lab/tree/' + path; } + }, + shiny: { + title: 'Shiny', + generateUrlPath: function (path) { + // jupyter-shiny-proxy requires everything to end with a trailing slash + if (!path.endsWith("/")) { + path = path + "/"; + } + return 'shiny/' + path; + } + }, + rstudio: { + title: 'RStudio', + generateUrlPath: function (path) { return 'rstudio/'; } + } +} + +function changeTab(div) { + var hub = document.getElementById("hub"); + var hub_help_text = document.getElementById("hub-help-text"); + var env_repo = document.getElementById("repo"); + var env_repo_branch = document.getElementById("branch"); + var env_repo_help_text = document.getElementById("env-repo-help-text"); + var content_repo = document.getElementById("content-repo-group"); + var content_branch = document.getElementById("content-branch-group"); + var id = div.id; + + if (id.includes("binder")) { + hub.placeholder = "https://mybinder.org"; + hub.value = "https://mybinder.org"; + hub_help_text.hidden = true; + hub.labels[0].innerHTML = "BinderHub URL"; + env_repo.labels[0].innerHTML = "Git Environment Repository URL"; + env_repo_help_text.hidden = false; + env_repo_branch.required = true; + env_repo_branch.pattern = ".+"; + content_repo.hidden = false; + content_branch.hidden = false; + } else { + hub.placeholder = "https://hub.example.com"; + hub_help_text.hidden = false; + hub.labels[0].innerHTML = "JupyterHub URL"; + env_repo.labels[0].innerHTML = "Git Repository URL"; + env_repo_help_text.hidden = true; + env_repo_branch.required = false; + content_repo.hidden = true; + content_branch.hidden = true; + } +} + +/** + * Return name of directory git will clone given repo to. + * + * nbgitpuller needs to redirect users to *inside* the directory it + * just cloned. We copy the logic git itself uses to determine that. + * See https://github.com/git/git/blob/1c52ecf4ba0f4f7af72775695fee653f50737c71/builtin/clone.c#L276 + */ +function generateCloneDirectoryName(gitCloneUrl) { + var lastPart = gitCloneUrl.split('/').slice(-1)[0]; + return lastPart.split(':').slice(-1)[0].replace(/(\.git|\.bundle)?/, ''); +} + +function displayLink() { + var form = document.getElementById('linkgenerator'); + + form.classList.add('was-validated'); + if (form.checkValidity()) { + var hubUrl = document.getElementById('hub').value; + var repoUrl = document.getElementById('repo').value; + var branch = document.getElementById('branch').value; + var contentRepoUrl = document.getElementById('content-repo').value; + var contentRepoBranch = document.getElementById('content-branch').value; + var filePath = document.getElementById('filepath').value; + var appName = form.querySelector('input[name="app"]:checked').value; + var activeTab = document.querySelector(".nav-link.active").id; + + if (appName === 'custom') { + var urlPath = document.getElementById('urlpath').value; + } else { + var repoName = generateCloneDirectoryName(repoUrl); + var urlPath; + if (activeTab === "tab-auth-binder") { + var contentRepoName = new URL(contentRepoUrl).pathname.split('/').pop().replace(/\.git$/, ''); + urlPath = apps[appName].generateUrlPath(contentRepoName + '/' + filePath); + } else { + urlPath = apps[appName].generateUrlPath(repoName + '/' + filePath); + } + } + + if (activeTab === "tab-auth-default") { + document.getElementById('default-link').value = generateRegularUrl( + hubUrl, urlPath, repoUrl, branch + ); + } else if (activeTab === "tab-auth-canvas"){ + document.getElementById('canvas-link').value = generateCanvasUrl( + hubUrl, urlPath, repoUrl, branch + ); + } else if (activeTab === "tab-auth-binder"){ + // FIXME: userName parsing using new URL(...) assumes a + // HTTP based repoUrl. Does it make sense to create a + // BinderHub link for SSH URLs? Then let's fix this parsing. + var userName = new URL(repoUrl).pathname.split('/')[1]; + document.getElementById('binder-link').value = generateBinderUrl( + hubUrl, userName, repoName, branch, urlPath, contentRepoUrl, contentRepoBranch + ); + } + } +} +function populateFromQueryString() { + // preseed values if specified in the url + var params = new URLSearchParams(window.location.search); + // Parameters are read from query string, and fields are set to them + var allowedParams = ['hub', 'repo', 'content-repo', 'branch', 'app', 'urlpath']; + if (params.has("urlpath")) { + // setting urlpath implies a custom app + document.getElementById('app-custom').checked = true; + } + for (var i = 0; i < allowedParams.length; i++) { + var param = allowedParams[i]; + if (params.has(param)) { + if ((param === 'app') && !params.has("urlpath")) { + radioId = 'app-' + params.get(param).toLowerCase(); + document.getElementById(radioId).checked = true; + } else { + document.getElementById(param).value = params.get(param); + } + } + } +} + +/** + * Main loop of the program. + * + * Called whenever any state changes (input received, page loaded, etc). + * Should turn on / off elements based only on current state, and display the link + * + * Sort of react-ish. + */ +function render() { + var form = document.getElementById('linkgenerator'); + var appName = form.querySelector('input[name="app"]:checked').value; + + if (appName == 'custom') { + document.getElementById('urlpath').disabled = false; + document.getElementById('filepath').disabled = true; + } else { + document.getElementById('urlpath').disabled = true; + + var app = apps[appName]; + if (!app.generateUrlPath) { + document.getElementById('filepath').disabled = true; + } else { + document.getElementById('filepath').disabled = false; + } + } + displayLink(); +} + +/** + * Entry point + */ +function main() { + // Hook up any changes in form elements to call render() + document.querySelectorAll('#linkgenerator input[type="radio"]').forEach( + function (element) { + element.addEventListener('change', render); + } + ) + document.querySelectorAll('#linkgenerator input[type="text"], #linkgenerator input[type="url"]').forEach( + function (element) { + element.addEventListener('input', render); + } + ) + + populateFromQueryString(); + + // Activate tabs based on search parameters + var params = new URL(window.location).searchParams; + if (params.get("tab")) { + if (params.get("tab") === "binder") { + $("#tab-auth-binder").click() + } else if (params.get("tab") === "canvas") { + $("#tab-auth-canvas").click() + } + } + + // Do an initial render, to make sure our disabled / enabled properties are correctly set + render(); +} + +window.onload = main; diff --git a/nbgitpuller/docs/_static/nbpuller.gif b/nbgitpuller/docs/_static/nbpuller.gif new file mode 100644 index 0000000000000000000000000000000000000000..9ccbe76d4ee260dbbcdca0007f382a49e2166735 Binary files /dev/null and b/nbgitpuller/docs/_static/nbpuller.gif differ diff --git a/nbgitpuller/docs/_templates/layout.html b/nbgitpuller/docs/_templates/layout.html new file mode 100644 index 0000000000000000000000000000000000000000..5598856ab5c56e59071dac002bf1bbdfdbb29fff --- /dev/null +++ b/nbgitpuller/docs/_templates/layout.html @@ -0,0 +1,14 @@ +{%- extends "sphinx_book_theme/layout.html" %} + + +{% block extrahead %} +{% if pagename == 'link' %} + +{% endif %} +{{ super() }} +{% endblock %} diff --git a/nbgitpuller/docs/conf.py b/nbgitpuller/docs/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..66fb9cb2580fae75fd2859d420db2a998a90c30e --- /dev/null +++ b/nbgitpuller/docs/conf.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +github_doc_root = "https://github.com/rtfd/recommonmark/tree/master/doc/" + + +def setup(app): + app.add_stylesheet("custom.css") + app.add_javascript("link_gen/link.js") + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "myst_parser", + "sphinx.ext.intersphinx", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: + +source_suffix = [".rst", ".md"] + + +# The root toctree document. +root_doc = master_doc = "index" + +# General information about the project. +project = "nbgitpuller" +copyright = "2017, The nbgitpuller Team" +author = "The nbgitpuller Team" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = "0.1b" +# The full version, including alpha/beta/rc tags. +release = "0.1b" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +html_sidebars = {"**": ["globaltoc.html", "relations.html", "searchbox.html"]} + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_book_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +html_context = { + "github_user": "jupyterhub", + "github_repo": "nbgitpuller", + "github_version": "main", + "doc_path": "doc", + "source_suffix": source_suffix, +} + +html_theme_options = { + "repository_url": "https://github.com/jupyterhub/nbgitpuller", + "use_issues_button": True, + "use_repository_button": True +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "nbgitpullerdoc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + root_doc, + "nbgitpuller.tex", + "nbgitpuller Documentation", + "The nbgitpuller Team", + "manual", + ) +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(root_doc, "nbgitpuller", "nbgitpuller Documentation", [author], 1)] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + root_doc, + "nbgitpuller", + "nbgitpuller Documentation", + author, + "nbgitpuller", + "One line description of project.", + "Miscellaneous", + ) +] diff --git a/nbgitpuller/docs/contributing.md b/nbgitpuller/docs/contributing.md new file mode 100644 index 0000000000000000000000000000000000000000..0df9b8f821f8cb2827fcb9cb0a4908f786a9f14c --- /dev/null +++ b/nbgitpuller/docs/contributing.md @@ -0,0 +1,133 @@ +# Contributing + +## Setup + +nbgitpuller is a jupyter extension that works with both the +[classic Notebook Server](https://jupyter-notebook.readthedocs.io/en/stable/extending/handlers.html), +and the newer [Jupyter Server](https://jupyter-server.readthedocs.io/en/latest/operators/configuring-extensions.html). +Hence, nbgitpuller can be developed locally without needing a JupyterHub. + +1. Fork the nbgitpuller repository and `git clone` it to your local computer. + +2. Inside the nbgitpuller clone on your local machine, setup a virtual + environment to do development in + + ```bash + python3 -m venv venv + source venv/bin/activate + ``` + +3. Install development time dependencies in this virtual environment + + ```bash + pip install -r dev-requirements.txt + ``` + +4. Install nbgitpuller with its dependencies in this virtual environment + + ```bash + pip install -e . + ``` + +5. Install the NodeJS dependencies from package.json. + + ```bash + npm install + ``` + +6. Create the JS and CSS bundles. + + ```bash + npm run webpack + ``` + +7. Enable the nbgitpuller extension: + * as a jupyter serverextension (classic Notebook Server extension) + + ```bash + jupyter serverextension enable --sys-prefix nbgitpuller + ``` + * as a jupyter server extension + ```bash + jupyter server extension enable --sys-prefix nbgitpuller + ``` + +8. Start the notebook server: + + * You can either start the classical Notebook server. + This will open the classic notebook in your web + browser, and automatically authenticate you as a side effect. + + ```bash + jupyter notebook + ``` + + * Or you can start the new Jupyter Server. + ```bash + jupyter server + ``` + This won't open any notebook interface, unless you don't enable one + ([`nbclassic`](https://github.com/jupyterlab/nbclassic) or [`jupyterlab`](https://github.com/jupyterlab/jupyterlab)) + as a jupyter server extension. + ```bash + jupyter server extension enable --sys-prefix nbclassic + ``` + or + ```bash + jupyter server extension enable --sys-prefix jupyterlab + ``` + +9. You can now test nbgitpuller locally, by hitting the `/git-pull` url with any + of the [URL query parameters](topic/url-options.rst). For example, to pull the + [data-8/textbook](https://github.com/data-8/textbook) repository's `gh-pages` + branch, you can use the following URL: + + ``` + http://localhost:8888/git-pull?repo=https://github.com/data-8/textbook&branch=gh-pages + ``` + +10. If you make changes to nbgitpuller's python code, you need to restart the `jupyter notebook` + process (started in step 5) to see your changes take effect. This is not needed if + you are only working on the javascript or css. + +## Running the flake8 linter + +[flake8](https://flake8.pycqa.org/en/latest/) is used to validate python coding style. The +flake8 config is in `.flake8`, and is not super strict. You should be able to run +`flake8` in the root directory of the repository to get a list of issues to be fixed. + +## Running tests + +[pytest](https://docs.pytest.org/) is used to run unit and integration tests, +under the `tests/` directory. If you add new functionality, you should also add +tests to cover it. You can run the tests locally with `py.test tests/` + +## Building documentation + +[sphinx](https://www.sphinx-doc.org/) is used to write and maintain documentation, under +the `docs/` directory. If you add any new functionality, you should write documentaiton +for it as well. A mix of [reStructuredText](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html) +and [MyST Markdown](https://myst-parser.readthedocs.io) is used to write our documentation, +although we would like to migrate purely to MyST markdown in the future. + +1. Install the packages needed to build the documentation + + ```bash + pip install -r docs/doc-requirements.txt + ``` + +2. Build the documentation by using `make` inside the `docs` folder. This will + internally call `sphinx` + + ```bash + cd docs + make html + ``` + +3. Preview the documentation by opening `_build/html/index.html` file in + your browser. From inside the `docs` folder, you can run either + `open _build/html/index.html` (on MacOS) or `xdg-open _build/html/index.html` + to quickly open the file in the browser. + +4. You can run `make html` again after making further changes to see their + effects. diff --git a/nbgitpuller/docs/doc-requirements.txt b/nbgitpuller/docs/doc-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8c4759b68b34db3c6d4c71a333c0075e11ad56d --- /dev/null +++ b/nbgitpuller/docs/doc-requirements.txt @@ -0,0 +1,3 @@ +myst_parser +sphinx_copybutton +sphinx-book-theme diff --git a/nbgitpuller/docs/index.rst b/nbgitpuller/docs/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..ef65d01fdb246dc5d9bfbd34594aa30c185eba76 --- /dev/null +++ b/nbgitpuller/docs/index.rst @@ -0,0 +1,96 @@ +=========== +nbgitpuller +=========== + +``nbgitpuller`` lets you distribute content in a git repository to your +students by having them click a simple link. :ref:`Automatic, opinioned +conflict resolution ` ensures that your students are +never exposed to ``git`` directly. It is primarily used with a JupyterHub, +but can also work on students' local laptops. + +.. image:: _static/nbpuller.gif + +When to use nbgitpuller? +======================== + +You should use nbgitpuller when: + +#. You are running a JupyterHub for a class & want an easy way to distribute + materials to your students without them having to understand what git is. +#. You have a different out of band method for collecting completed + assignments / notebooks from students, since they can not just 'push it + back' via git. + +You should **not** use nbgitpuller when: + +#. You are an instructor using a JupyterHub / running notebooks locally to + create materials and push them to a git repository. You should just use + git directly, since the assumptions and design of nbgitpuller **will** + surprise you in unexpected ways if you are pushing with git but pulling + with nbgitpuller. +#. Your students are performing manual git operations on the git repository + cloned as well as using nbgitpuller. Mixing manual git operations + + automatic nbgitpuller operations is going to cause surprises on an ongoing + basis, and should be avoided. + +Installation +============ + +If you already have a JupyterHub, you can follow :ref:`these installation +instructions ` to install nbgitpuller there. They should also +work for installation on a local Jupyter Notebook installation without +JupyterHub. + +If you do *not* have a JupyterHub, we recommend trying out `The Littlest +JupyterHub `_ to set one up. It comes built +in with nbgitpuller. + +Using nbgitpuller as an instructor +================================== + +Once installed, you create a specially crafted web link (called +*nbgitpuller links*) and send to your students via any method you like - +course website, LMS, email, etc. This link will contain at least the +following information: + +#. The location of the JupyterHub you are sending them to. +#. The git repository where you have published your content. +#. Optionally, a particular file or directory you want to automatically + open for your students once the repository has been synchronized. Note the entire repository will be copied, not just the specified file. + +The first time a particular student clicks the link, a local copy of the +repository is made for the student. On successive clicks, the latest version +of the remote repository is fetched, and merged automatically with the +student's local copy using a :ref:`series of rules ` +that ensure students never get merge conflicts. + +You can generate such *nbgitpuller links* with the `generator +`_. + +There is also a video showing you how to use nbgitpuller + +.. raw:: html + + + +If you are interested in the details of available options when creating +the link, we have a :ref:`list of options ` as well. + +Full Contents +============= + +.. toctree:: + :maxdepth: 2 + + install + contributing + topic/automatic-merging + topic/url-options + topic/repo-best-practices + link diff --git a/nbgitpuller/docs/install.rst b/nbgitpuller/docs/install.rst new file mode 100644 index 0000000000000000000000000000000000000000..98f42f080411b6357409a50ffe288b8cb0414817 --- /dev/null +++ b/nbgitpuller/docs/install.rst @@ -0,0 +1,26 @@ +.. _install: + +============ +Installation +============ + +You can install ``nbgitpuller`` from PyPI with ``pip`` in the same +environment where your jupyter notebook package is installed. + +.. code:: bash + + pip install nbgitpuller + +Troubleshooting +=============== + +nbgitpuller link shows `404 Not Found` +-------------------------------------- + +If you are on an old version of Jupyter Notebook, you might get a `404 Not Found` +error when trying to access an nbgitpuller link. You might need to manually enable +the server extension that handles nbgitpuller. + +.. code:: bash + + jupyter serverextension enable nbgitpuller --sys-prefix diff --git a/nbgitpuller/docs/link.rst b/nbgitpuller/docs/link.rst new file mode 100644 index 0000000000000000000000000000000000000000..a39395d3420c0666922fa3c5dab646dfb9844e6c --- /dev/null +++ b/nbgitpuller/docs/link.rst @@ -0,0 +1,188 @@ +nbgitpuller link generator +========================== + +Use the following form to create your own ``nbgitpuller`` links. + +.. raw:: html + +
+
+ +
+ + + +
+
+ +
+
+ +
+
+ +
+
+ +
+ +
+ +
+ +
+ Must be a valid web URL +
+ + The JupyterHub to send users to. + nbgitpuller must be installed in this hub. + +
+
+ +
+ +
+ +
+ Must be a valid git URL +
+ +
+
+
+
+ branch +
+ + + Use main instead of master for + + new GitHub repositories + +
+ Must specify a branch name +
+
+
+
+ + + +
+ +
+ + + This file or directory from within the repo will open when user clicks the link. + +
+
+ +
+
+ + + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + +
+
+
+ +
+
+


+ + +**Pre-populating some fields in the link generator** + +You can pre-populate some fields in order to make it easier for some +users to create their own links. To do so, use the following URL +parameters **when accessing this page**: + +* ``hub`` is the URL of a JupyterHub +* ``repo`` is the URL of a github repository to which you're linking +* ``branch`` is the branch you wish to pull from the Repository + +For example, the following URL will pre-populate the form with the +UC Berkeley DataHub as the JupyterHub:: + + https://jupyterhub.github.io/nbgitpuller/link?hub=https://datahub.berkeley.edu + + +**Activating a tab when someone lands on this page** + +You can also activate one of the tabs in the form above by default when a user lands +on this page. To do so, use the ``tab=`` REST parameter. Here are the possible values: + +* ``?tab=binder`` - activates the Binder tab +* ``?tab=canvas`` - activates the Canvas tab. diff --git a/nbgitpuller/docs/topic/automatic-merging.rst b/nbgitpuller/docs/topic/automatic-merging.rst new file mode 100644 index 0000000000000000000000000000000000000000..5aa6e4ec587d2795abe03b35e3c361977f3601f4 --- /dev/null +++ b/nbgitpuller/docs/topic/automatic-merging.rst @@ -0,0 +1,61 @@ +.. _topic/automatic-merging: + +========================== +Automatic Merging Behavior +========================== + +``nbgitpuller`` tries to make sure the end user who clicked the link +**never** has to manually interact with the git repo. This requires us to +make some opinionated choices on how we handle various cases where both the +student (end user) and instructor (author of the repo) repo have modified the +repository. + +Here, we describe how we handle the various possible cases each time the +student clicks the nbgitpuller link. + +Case 1: The instructor changed a file that the student has not changed +====================================================================== + +The student's changes are left alone, and the instructor's changes are pulled +in to the local copy. Most common case. This is also what happens when the +instructor adds a new file / directory. + +Case 2: Student & instructor changed different lines in same file +================================================================= + +Very similar to case 1 - the student's changes are left alone, and the +instructor's changes are merged in to the existing local file. + +Case 3: Student & instructor change same lines in same file +=========================================================== + +In this case, we **always keep the student's changes**. We want to never +accidentally lose a student's changes - ``nbgitpuller`` will not eat your +homework. + +Case 4: Student deletes file locally, but instructor doesn't +============================================================ + +If the student has deleted a file locally, but the file is still present in +the remote repo, the file from the remote repo is pulled into the student's +directory. This enables the use case where a student wants to 'start over' +a file after having made many changes to it. They can simply delete the file, +click the nbgitpuller link again, and get a fresh copy. + +Case 5: Student creates file manually, but instructor adds file with same name +============================================================================== + +As an example, let's say the student manually creates a file named +``Untitled141.ipynb`` in the directory where nbgitpuller has pulled a +repository. At some point afterwards, the instructor creates a file *also* +named ``Untitled141.ipynb`` and pushes it to the repo. + +When the student clicks the nbgitpuller link next, we want to make sure we +don't destroy the student's work. Since they were created in two different +places, the likelihood of them being mergeable is low. So we **rename** the +student's file, and pull the instructor's file. So the student's +``Untitled141.ipynb`` file will be renamed to +``Untitled141_.ipynb``, and the instructor's file will be kept at +``Untitled141.ipynb``. + +This is a fairly rare case in our experience. diff --git a/nbgitpuller/docs/topic/repo-best-practices.md b/nbgitpuller/docs/topic/repo-best-practices.md new file mode 100644 index 0000000000000000000000000000000000000000..025b6dfb88619ebd256034a03823fe8dabc45e34 --- /dev/null +++ b/nbgitpuller/docs/topic/repo-best-practices.md @@ -0,0 +1,41 @@ +# Content git repository best practices + +Sometimes, git's flexibility can lead to repositories that cause issues +when used with nbgitpuller. Here are some recommendations to make your +nbgitpuller experience smoother. + + +## Never force push + +Never use `--force` or `--force-with-lease` when pushing to your repositories. +This is general good git practice, and unless you have [fairly deep +understanding](https://xkcd.com/1597/) of how git works, it might screw up some +of your users' local repositories beyond repair. + +If you are using GitHub, you should enable [protected branches](https://docs.github.com/en/github/administering-a-repository/about-protected-branches) +to prevent accidental force pushes. + +## Prevent your repos from becoming huge + +Larger git repos increase chances of timeouts and other intermittent failures +that will be difficult to debug. They might leave your git repo in strange states +too - contents fetched but not checked out, half-fetched, etc. Try and keep it small - +under 100MB is great, under 1G is ok, but anything more is probably asking for trouble. + +Large datasets are the biggest reason for increasing repository sizes. Try distribute +datasets some other way, use a subset of data, or compress your data if you need to. + +## Don't add `.ipynb_checkpoints` (and similar files) to your git repo + +Jupyter uses a hidden `.ipynb_checkpoints` directory to temporarily autosave copies of the +notebook. If you accidentally commit your local computer's copy of this to the git repo, +it can cause hard to debug issues when students click nbgitpuller links. The students' +Jupyter Notebook servers in the JupyterHub will also generate `.ipynb_checkpoints` for +autosaving, and conflicts between these two can cause issues. Similar issues can happen +with other temporary, hidden files - like `.DS_Store`, `__pycache__`, etc. + +Adding `.ipynb_checkpoints` to your repo's `.gitignore` file will eliminate this +class of issues completely. `git add` and similar commands will no longer +accidentally include them in your repo. You can download this [python specific +gitignore](https://github.com/github/gitignore/blob/master/Python.gitignore) +file and put it in your repo as `.gitignore`, and it should take care of this. diff --git a/nbgitpuller/docs/topic/url-options.rst b/nbgitpuller/docs/topic/url-options.rst new file mode 100644 index 0000000000000000000000000000000000000000..200d3be14644885d265cb934e1fec9415d8387a3 --- /dev/null +++ b/nbgitpuller/docs/topic/url-options.rst @@ -0,0 +1,110 @@ +.. _topic/url-options: + +============================= +Options in an nbgitpuller URL +============================= + +.. note:: + + If you just want to generate an nbgitpuller link, we highly + recommend just using the `link generator `_ + +Most aspects of the nbgitpuller student experience can be configured +with various options in the nbgitpuller URL. This page documents +the various options available, and their behavior. + +``repo`` +======== + +The path to the git repository to be pulled from. This will accept +any parameter that can be passed to a ``git clone`` command. + +``branch`` +========== + +Branch in the git repo to pull from. Defaults to ``master``. + +``urlpath`` +=========== + +The URL to redirect the user to after synchronization has been complete. This +URL is primarily used to open a specific file or directory in a specific +application. This URL is interpreted relative to the base of the notebook +server. The URL to be specified depends on the application you want +the file to be opened in. + +.. warning:: + + ```` is relative to the directory the notebook + server was launched in - so the directory you see if you login to + JupyterHub regularly. This means you **must** include the name of + the local repository directory too, otherwise nbgitpuller can not + find the file. + + For example, if the repository you are cloning is + ``https://github.com/my-user/my-repository``, and the file you want + your students to see is ``index.ipynb``, then ```` + should be ``my-repository/index.ipynb``, **not** ``index.ipynb``. + + The `link generator `_ + takes care of all of this for you, so it is recommended to use that. + + +Classic Jupyter Notebook +------------------------ + +To open a notebook, file or directory in the classic Jupyter Notebook +interface, your pattern should be: ``/tree/``. + +JupyterLab +---------- + +To open a notebook, file or directory in the classic Jupyter Notebook +interface, your pattern should be: +``/lab/tree/%3Fautodecode``. + +The ``%3Fautodecode`` at the end makes sure you never get `a message +`_ about needing to +explicitly name a JupyterLab workspace. + +Shiny +----- + +To open a directory containing `shiny `_ files, +your pattern should be ``/shiny//``. The trailing +slash is important. + +RStudio +------- + +If you have RStudio installed and set up for use with your JupyterHub, +you can pass ``/rstudio`` to ``urlpath`` to open RStudio after the +repo has been pulled. You can not have RStudio open a specific file +or directory, unfortunately. + +``depth`` +========= + +How deep to clone the git repo on initial pull. By default, the +entire history of the git repository is pulled. This might be +slow if your git repository is large. You can set this to 1 to +pull only the latest commit on initial pull. + +Only explicitly set this if you are actively having performance +problems. + + +``targetPath`` +============== + +Where to place the repository when it is cloned. +By default, Git repositories are cloned into the default working directory. +You can specify a different parent directory for the clone by setting the environment variable ``NBGITPULLER_PARENTPATH``, this should be relative to the working directory. +If you require full control over the destination directory, or want to set the directory at runtime in the nbgitpuller link use this parameter. + + +Deprecated parameters +===================== + +The following parameters are currently deprecated, and will be removed in +a future version: ``subpath``, ``app``. diff --git a/nbgitpuller/nbgitpuller/__init__.py b/nbgitpuller/nbgitpuller/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b594128653e75cd874bd3627f3b841f821cb79e3 --- /dev/null +++ b/nbgitpuller/nbgitpuller/__init__.py @@ -0,0 +1,32 @@ +from .version import __version__ # noqa +from .handlers import SyncHandler, UIHandler, LegacyInteractRedirectHandler, LegacyGitSyncRedirectHandler +from .pull import GitPuller # noqa +from notebook.utils import url_path_join +from tornado.web import StaticFileHandler +import os + + +def _jupyter_server_extension_paths(): + return [{ + 'module': 'nbgitpuller', + }] + + +def load_jupyter_server_extension(nbapp): + web_app = nbapp.web_app + base_url = url_path_join(web_app.settings['base_url'], 'git-pull') + handlers = [ + (url_path_join(base_url, 'api'), SyncHandler), + (base_url, UIHandler), + (url_path_join(web_app.settings['base_url'], 'git-sync'), LegacyGitSyncRedirectHandler), + (url_path_join(web_app.settings['base_url'], 'interact'), LegacyInteractRedirectHandler), + ( + url_path_join(base_url, 'static', '(.*)'), + StaticFileHandler, + {'path': os.path.join(os.path.dirname(__file__), 'static')} + ) + ] + web_app.settings['nbapp'] = nbapp + web_app.add_handlers('.*', handlers) + +_load_jupyter_server_extension = load_jupyter_server_extension diff --git a/nbgitpuller/nbgitpuller/etc/jupyter_notebook_config.d/nbgitpuller.json b/nbgitpuller/nbgitpuller/etc/jupyter_notebook_config.d/nbgitpuller.json new file mode 100644 index 0000000000000000000000000000000000000000..ccfc3f5d72a9df6e576ab5027c15592da32a8000 --- /dev/null +++ b/nbgitpuller/nbgitpuller/etc/jupyter_notebook_config.d/nbgitpuller.json @@ -0,0 +1,8 @@ +{ + "NotebookApp": { + "nbserver_extensions": { + "nbgitpuller": true + } + } +} + diff --git a/nbgitpuller/nbgitpuller/etc/jupyter_server_config.d/nbgitpuller.json b/nbgitpuller/nbgitpuller/etc/jupyter_server_config.d/nbgitpuller.json new file mode 100644 index 0000000000000000000000000000000000000000..e395847200431d8dade30bd8c5a3b43e10318c0f --- /dev/null +++ b/nbgitpuller/nbgitpuller/etc/jupyter_server_config.d/nbgitpuller.json @@ -0,0 +1,8 @@ +{ + "ServerApp": { + "jpserver_extensions": { + "nbgitpuller": true + } + } +} + diff --git a/nbgitpuller/nbgitpuller/handlers.py b/nbgitpuller/nbgitpuller/handlers.py new file mode 100644 index 0000000000000000000000000000000000000000..ae72afd9602caf160bc9ab17717d8262f90da79a --- /dev/null +++ b/nbgitpuller/nbgitpuller/handlers.py @@ -0,0 +1,216 @@ +from tornado import gen, web, locks +import traceback +import urllib.parse + +from notebook.base.handlers import IPythonHandler +import threading +import json +import os +from queue import Queue, Empty +import jinja2 + +from .pull import GitPuller +from .version import __version__ +from .wget import RequestRepoRawFile + + +class SyncHandler(IPythonHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # We use this lock to make sure that only one sync operation + # can be happening at a time. Git doesn't like concurrent use! + if 'git_lock' not in self.settings: + self.settings['git_lock'] = locks.Lock() + + @property + def git_lock(self): + return self.settings['git_lock'] + + @gen.coroutine + def emit(self, data): + if type(data) is not str: + serialized_data = json.dumps(data) + if 'output' in data: + self.log.info(data['output'].rstrip()) + else: + serialized_data = data + self.log.info(data) + self.write('data: {}\n\n'.format(serialized_data)) + yield self.flush() + + @web.authenticated + @gen.coroutine + def get(self): + try: + yield self.git_lock.acquire(1) + except gen.TimeoutError: + self.emit({ + 'phase': 'error', + 'message': 'Another git operations is currently running, try again in a few minutes' + }) + return + + try: + repo = self.get_argument('repo') + branch = self.get_argument('branch', None) + depth = self.get_argument('depth', None) + if depth: + depth = int(depth) + # The default working directory is the directory from which Jupyter + # server is launched, which is not the same as the root notebook + # directory assuming either --notebook-dir= is used from the + # command line or c.NotebookApp.notebook_dir is set in the jupyter + # configuration. This line assures that all repos are cloned + # relative to server_root_dir/, + # so that all repos are always in scope after cloning. Sometimes + # server_root_dir will include things like `~` and so the path + # must be expanded. + repo_parent_dir = os.path.join(os.path.expanduser(self.settings['server_root_dir']), + os.getenv('NBGITPULLER_PARENTPATH', '')) + repo_dir = os.path.join(repo_parent_dir, self.get_argument('targetpath', repo.split('/')[-1])) + + # We gonna send out event streams! + self.set_header('content-type', 'text/event-stream') + self.set_header('cache-control', 'no-cache') + + gp = GitPuller(repo, repo_dir, branch=branch, depth=depth, parent=self.settings['nbapp']) + + q = Queue() + + def pull(): + try: + for line in gp.pull(): + q.put_nowait(line) + # Sentinel when we're done + q.put_nowait(None) + except Exception as e: + q.put_nowait(e) + raise e + self.gp_thread = threading.Thread(target=pull) + + self.gp_thread.start() + + while True: + try: + progress = q.get_nowait() + except Empty: + yield gen.sleep(0.5) + continue + if progress is None: + break + if isinstance(progress, Exception): + self.emit({ + 'phase': 'error', + 'message': str(progress), + 'output': '\n'.join([ + line.strip() + for line in traceback.format_exception( + type(progress), progress, progress.__traceback__ + ) + ]) + }) + return + + self.emit({'output': progress, 'phase': 'syncing'}) + + self.emit({'phase': 'finished'}) + except Exception as e: + self.emit({ + 'phase': 'error', + 'message': str(e), + 'output': '\n'.join([ + line.strip() + for line in traceback.format_exception( + type(e), e, e.__traceback__ + ) + ]) + }) + finally: + self.git_lock.release() + + +class UIHandler(IPythonHandler): + def initialize(self): + super().initialize() + # FIXME: Is this really the best way to use jinja2 here? + # I can't seem to get the jinja2 env in the base handler to + # actually load templates from arbitrary paths ugh. + jinja2_env = self.settings['jinja2_env'] + jinja2_env.loader = jinja2.ChoiceLoader([ + jinja2_env.loader, + jinja2.FileSystemLoader( + os.path.join(os.path.dirname(__file__), 'templates') + ) + ]) + + @web.authenticated + @gen.coroutine + def get(self): + app_env = os.getenv('NBGITPULLER_APP', default='notebook') + + repo = self.get_argument('repo') + branch = self.get_argument('branch', None) + depth = self.get_argument('depth', None) + urlPath = self.get_argument('urlpath', None) or \ + self.get_argument('urlPath', None) + subPath = self.get_argument('subpath', None) or \ + self.get_argument('subPath', '.') + app = self.get_argument('app', app_env) + parent_reldir = os.getenv('NBGITPULLER_PARENTPATH', '') + targetpath = self.get_argument('targetpath', None) or \ + self.get_argument('targetPath', repo.split('/')[-1]) + if(urlPath.endswith('.ipynb')): + # 添加获取文件 和跳转的逻辑 + rrrf = RequestRepoRawFile() + path = rrrf.wgetFile(repo, branch, urlPath) + else: + if urlPath: + path = urlPath + else: + path = os.path.join(parent_reldir, targetpath, subPath) + if app.lower() == 'lab': + path = 'lab/tree/' + path + elif path.lower().endswith('.ipynb'): + path = 'notebooks/' + path + else: + path = 'tree/' + path + + self.write( + self.render_template( + 'status.html', + repo=repo, branch=branch, path=path, depth=depth, targetpath=targetpath, version=__version__ + )) + self.flush() + + +class LegacyGitSyncRedirectHandler(IPythonHandler): + @web.authenticated + @gen.coroutine + def get(self): + new_url = '{base}git-pull?{query}'.format( + base=self.base_url, + query=self.request.query + ) + self.redirect(new_url) + + +class LegacyInteractRedirectHandler(IPythonHandler): + @web.authenticated + @gen.coroutine + def get(self): + repo = self.get_argument('repo') + account = self.get_argument('account', 'data-8') + repo_url = 'https://github.com/{account}/{repo}'.format(account=account, repo=repo) + query = { + 'repo': repo_url, + # branch & subPath are optional + 'branch': self.get_argument('branch', 'gh-pages'), + 'subPath': self.get_argument('path', '.') + } + new_url = '{base}git-pull?{query}'.format( + base=self.base_url, + query=urllib.parse.urlencode(query) + ) + + self.redirect(new_url) diff --git a/nbgitpuller/nbgitpuller/pull.py b/nbgitpuller/nbgitpuller/pull.py new file mode 100644 index 0000000000000000000000000000000000000000..cc18ac97ae48615c566705d4d659f3250b7705d4 --- /dev/null +++ b/nbgitpuller/nbgitpuller/pull.py @@ -0,0 +1,319 @@ +import os +import subprocess +import logging +import time +import argparse +import datetime +from traitlets import Integer, default +from traitlets.config import Configurable +from functools import partial + + +def execute_cmd(cmd, **kwargs): + """ + Call given command, yielding output line by line + """ + yield '$ {}\n'.format(' '.join(cmd)) + kwargs['stdout'] = subprocess.PIPE + kwargs['stderr'] = subprocess.STDOUT + + proc = subprocess.Popen(cmd, **kwargs) + + # Capture output for logging. + # Each line will be yielded as text. + # This should behave the same as .readline(), but splits on `\r` OR `\n`, + # not just `\n`. + buf = [] + + def flush(): + line = b''.join(buf).decode('utf8', 'replace') + buf[:] = [] + return line + + c_last = '' + try: + for c in iter(partial(proc.stdout.read, 1), b''): + if c_last == b'\r' and buf and c != b'\n': + yield flush() + buf.append(c) + if c == b'\n': + yield flush() + c_last = c + finally: + ret = proc.wait() + if ret != 0: + raise subprocess.CalledProcessError(ret, cmd) + + +class GitPuller(Configurable): + depth = Integer( + config=True, + help=""" + Depth (ie, commit count) of clone operations. Set this to 0 to make a + full depth clone. + + Defaults to the value of the environment variable NBGITPULLER_DEPTH, or + 1 if the the environment variable isn't set. + """ + ) + + @default('depth') + def _depth_default(self): + """This is a workaround for setting the same default directly in the + definition of the traitlet above. Without it, the test fails because a + change in the environment variable has no impact. I think this is a + consequence of the tests not starting with a totally clean environment + where the GitPuller class hadn't been loaded already.""" + return int(os.environ.get('NBGITPULLER_DEPTH', 1)) + + def __init__(self, git_url, repo_dir, **kwargs): + assert git_url + + self.git_url = git_url + self.branch_name = kwargs.pop("branch") + + if self.branch_name is None: + self.branch_name = self.resolve_default_branch() + elif not self.branch_exists(self.branch_name): + raise ValueError(f"Branch: {self.branch_name} -- not found in repo: {self.git_url}") + + self.repo_dir = repo_dir + newargs = {k: v for k, v in kwargs.items() if v is not None} + super(GitPuller, self).__init__(**newargs) + + def branch_exists(self, branch): + """ + This checks to make sure the branch we are told to access + exists in the repo + """ + try: + heads = subprocess.run( + ["git", "ls-remote", "--heads", "--", self.git_url], + capture_output=True, + text=True, + check=True + ) + tags = subprocess.run( + ["git", "ls-remote", "--tags", "--", self.git_url], + capture_output=True, + text=True, + check=True + ) + lines = heads.stdout.splitlines() + tags.stdout.splitlines() + branches = [] + for line in lines: + _, ref = line.split() + refs, heads, branch_name = ref.split("/", 2) + branches.append(branch_name) + return branch in branches + except subprocess.CalledProcessError: + m = f"Problem accessing list of branches and/or tags: {self.git_url}" + logging.exception(m) + raise ValueError(m) + + def resolve_default_branch(self): + """ + This will resolve the default branch of the repo in + the case where the branch given does not exist + """ + try: + head_branch = subprocess.run( + ["git", "ls-remote", "--symref", "--", self.git_url, "HEAD"], + capture_output=True, + text=True, + check=True + ) + for line in head_branch.stdout.splitlines(): + if line.startswith("ref:"): + # line resembles --> ref: refs/heads/main HEAD + _, ref, head = line.split() + refs, heads, branch_name = ref.split("/", 2) + return branch_name + raise ValueError(f"default branch not found in {self.git_url}") + except subprocess.CalledProcessError: + m = f"Problem accessing HEAD branch: {self.git_url}" + logging.exception(m) + raise ValueError(m) + + def pull(self): + """ + Pull selected repo from a remote git repository, + while preserving user changes + """ + if not os.path.exists(self.repo_dir): + yield from self.initialize_repo() + else: + yield from self.update() + + def initialize_repo(self): + """ + Clones repository + """ + logging.info('Repo {} doesn\'t exist. Cloning...'.format(self.repo_dir)) + clone_args = ['git', 'clone'] + if self.depth and self.depth > 0: + clone_args.extend(['--depth', str(self.depth)]) + clone_args.extend(['--branch', self.branch_name]) + clone_args.extend(["--", self.git_url, self.repo_dir]) + yield from execute_cmd(clone_args) + logging.info('Repo {} initialized'.format(self.repo_dir)) + + def reset_deleted_files(self): + """ + Runs the equivalent of git checkout -- for each file that was + deleted. This allows us to delete a file, hit an interact link, then get a + clean version of the file again. + """ + + yield from self.ensure_lock() + deleted_files = subprocess.check_output([ + 'git', 'ls-files', '--deleted', '-z' + ], cwd=self.repo_dir).decode().strip().split('\0') + + for filename in deleted_files: + if filename: # Filter out empty lines + yield from execute_cmd(['git', 'checkout', 'origin/{}'.format(self.branch_name), '--', filename], cwd=self.repo_dir) + + def repo_is_dirty(self): + """ + Return true if repo is dirty + """ + try: + subprocess.check_call(['git', 'diff-files', '--quiet'], cwd=self.repo_dir) + # Return code is 0 + return False + except subprocess.CalledProcessError: + return True + + def update_remotes(self): + """ + Do a git fetch so our remotes are up to date + """ + yield from execute_cmd(['git', 'fetch'], cwd=self.repo_dir) + + def find_upstream_changed(self, kind): + """ + Return list of files that have been changed upstream belonging to a particular kind of change + """ + output = subprocess.check_output([ + 'git', 'log', '..origin/{}'.format(self.branch_name), + '--oneline', '--name-status' + ], cwd=self.repo_dir).decode() + files = [] + for line in output.split('\n'): + if line.startswith(kind): + files.append(os.path.join(self.repo_dir, line.split('\t', 1)[1])) + + return files + + def ensure_lock(self): + """ + Make sure we have the .git/lock required to do modifications on the repo + + This must be called before any git commands that modify state. This isn't guaranteed + to be atomic, due to the nature of using files for locking. But it's the best we + can do right now. + """ + try: + lockpath = os.path.join(self.repo_dir, '.git', 'index.lock') + mtime = os.path.getmtime(lockpath) + # A lock file does exist + # If it's older than 10 minutes, we just assume it is stale and take over + # If not, we fail with an explicit error. + if time.time() - mtime > 600: + yield "Stale .git/index.lock found, attempting to remove" + os.remove(lockpath) + yield "Stale .git/index.lock removed" + else: + raise Exception('Recent .git/index.lock found, operation can not proceed. Try again in a few minutes.') + except FileNotFoundError: + # No lock is held by other processes, we are free to go + return + + def rename_local_untracked(self): + """ + Rename local untracked files that would require pulls + """ + # Find what files have been added! + new_upstream_files = self.find_upstream_changed('A') + for f in new_upstream_files: + if os.path.exists(f): + # If there's a file extension, put the timestamp before that + ts = datetime.datetime.now().strftime('__%Y%m%d%H%M%S') + path_head, path_tail = os.path.split(f) + path_tail = ts.join(os.path.splitext(path_tail)) + new_file_name = os.path.join(path_head, path_tail) + os.rename(f, new_file_name) + yield 'Renamed {} to {} to avoid conflict with upstream'.format(f, new_file_name) + + def update(self): + """ + Do the pulling if necessary + """ + # Fetch remotes, so we know we're dealing with latest remote + yield from self.update_remotes() + + # Rename local untracked files that might be overwritten by pull + yield from self.rename_local_untracked() + + # Reset local files that have been deleted. We don't actually expect users to + # delete something that's present upstream and expect to keep it. This prevents + # unnecessary conflicts, and also allows users to click the link again to get + # a fresh copy of a file they might have screwed up. + yield from self.reset_deleted_files() + + # If there are local changes, make a commit so we can do merges when pulling + # We also allow empty commits. On NFS (at least), sometimes repo_is_dirty returns a false + # positive, returning True even when there are no local changes (git diff-files seems to return + # bogus output?). While ideally that would not happen, allowing empty commits keeps us + # resilient to that issue. + # We explicitly set user info of the commits we are making, to keep that separate from + # whatever author info is set in system / repo config by the user. We pass '-c' to git + # itself (rather than to 'git commit') to temporarily set config variables. This is + # better than passing --author, since git treats author separately from committer. + if self.repo_is_dirty(): + yield from self.ensure_lock() + yield from execute_cmd([ + 'git', + '-c', 'user.email=nbgitpuller@nbgitpuller.link', + '-c', 'user.name=nbgitpuller', + 'commit', + '-am', 'Automatic commit by nbgitpuller', + '--allow-empty' + ], cwd=self.repo_dir) + + # Merge master into local! + yield from self.ensure_lock() + yield from execute_cmd([ + 'git', + '-c', 'user.email=nbgitpuller@nbgitpuller.link', + '-c', 'user.name=nbgitpuller', + 'merge', + '-Xours', 'origin/{}'.format(self.branch_name) + ], cwd=self.repo_dir) + + +def main(): + """ + Synchronizes a github repository with a local repository. + """ + logging.basicConfig( + format='[%(asctime)s] %(levelname)s -- %(message)s', + level=logging.DEBUG) + + parser = argparse.ArgumentParser(description='Synchronizes a github repository with a local repository.') + parser.add_argument('git_url', help='Url of the repo to sync') + parser.add_argument('branch_name', default=None, help='Branch of repo to sync', nargs='?') + parser.add_argument('repo_dir', default='.', help='Path to clone repo under', nargs='?') + args = parser.parse_args() + + for line in GitPuller( + args.git_url, + args.repo_dir, + branch=args.branch_name if args.branch_name else None + ).pull(): + print(line) + + +if __name__ == '__main__': + main() diff --git a/nbgitpuller/nbgitpuller/static/js/index.js b/nbgitpuller/nbgitpuller/static/js/index.js new file mode 100644 index 0000000000000000000000000000000000000000..6b2f7ac04825c6b46d2d40aa7621805a03ee6475 --- /dev/null +++ b/nbgitpuller/nbgitpuller/static/js/index.js @@ -0,0 +1,297 @@ +import { Terminal } from 'xterm'; +import { FitAddon } from 'xterm-addon-fit'; +import css from '../../../node_modules/xterm/css/xterm.css'; + +function GitSync(baseUrl, repo, branch, depth, targetpath, path) { + // Class that talks to the API backend & emits events as appropriate + this.baseUrl = baseUrl; + this.repo = repo; + this.branch = branch; + this.depth = depth; + this.targetpath = targetpath; + this.redirectUrl = baseUrl + path; + + this.callbacks = {}; +} + +GitSync.prototype.addHandler = function(event, cb) { + if (this.callbacks[event] == undefined) { + this.callbacks[event] = [cb]; + } else { + this.callbacks[event].push(cb); + } +}; + +GitSync.prototype._emit = function(event, data) { + if (this.callbacks[event] == undefined) { return; } + $.each(this.callbacks[event], function(i, ev) { + ev(data); + }); +}; + + +GitSync.prototype.start = function() { + if (this.path && this.path.endsWith('.ipynb')){ + that._emit('finished'); + }else{ + // Start git pulling handled by SyncHandler, declared in handlers.py + var syncUrlParams = { + repo: this.repo, + targetpath: this.targetpath + } + if (typeof this.depth !== 'undefined' && this.depth != undefined) { + syncUrlParams['depth'] = this.depth; + } + if (typeof this.branch !== 'undefined' && this.branch != undefined) { + syncUrlParams['branch'] = this.branch; + } + var syncUrl = this.baseUrl + 'git-pull/api?' + $.param(syncUrlParams); + + this.eventSource = new EventSource(syncUrl); + var that = this; + this.eventSource.addEventListener('message', function(ev) { + var data = JSON.parse(ev.data); + if (data.phase == 'finished' || data.phase == 'error') { + that.eventSource.close(); + } + that._emit(data.phase, data); + }); + this.eventSource.addEventListener('error', function(error) { + console.log(arguments); + that._emit('error', error); + }); + } +}; + +function GitSyncView(termSelector, progressSelector, termToggleSelector) { + // Class that encapsulates view rendering as much as possible + this.term = new Terminal({ + convertEol: true + }); + this.fit = new FitAddon(); + this.term.loadAddon(this.fit); + + this.visible = false; + this.$progress = $(progressSelector); + + this.$termToggle = $(termToggleSelector); + this.termSelector = termSelector; + + var that = this; + this.$termToggle.click(function() { + that.setTerminalVisibility(!that.visible); + }); +} + +GitSyncView.prototype.setTerminalVisibility = function(visible) { + if (visible) { + $(this.termSelector).parent().removeClass('hidden'); + } else { + $(this.termSelector).parent().addClass('hidden'); + } + this.visible = visible; + if (visible) { + // See https://github.com/jupyterhub/nbgitpuller/pull/46 on why this is here. + if (!this.term.element) { + this.term.open($(this.termSelector)[0]); + } + this.fit.fit(); + } + +} + +GitSyncView.prototype.setProgressValue = function(val) { + this.$progress.attr('aria-valuenow', val); + this.$progress.css('width', val + '%'); +}; + +GitSyncView.prototype.getProgressValue = function() { + return parseFloat(this.$progress.attr('aria-valuenow')); +}; + +GitSyncView.prototype.setProgressText = function(text) { + this.$progress.children('span').text(text); +}; + +GitSyncView.prototype.getProgressText = function() { + return this.$progress.children('span').text(); +}; + +GitSyncView.prototype.setProgressError = function(isError) { + if (isError) { + this.$progress.addClass('progress-bar-danger'); + } else { + this.$progress.removeClass('progress-bar-danger'); + } +}; + +var get_body_data = function(key) { + /** + * get a url-encoded item from body.data and decode it + * we should never have any encoded URLs anywhere else in code + * until we are building an actual request + */ + var val = $('body').data(key); + if (typeof val === 'undefined') + return val; + return decodeURIComponent(val); +}; + +var gs = new GitSync( + get_body_data('baseUrl'), + get_body_data('repo'), + get_body_data('branch'), + get_body_data('depth'), + get_body_data('targetpath'), + get_body_data('path') +); + +var gsv = new GitSyncView( + '#status-details', + '#status-panel-title', + '#status-panel-toggle' +); + +gs.addHandler('syncing', function(data) { + gsv.term.write(data.output); +}); +gs.addHandler('finished', function(data) { + progressTimers.forEach(function(timer) { clearInterval(timer); }); + gsv.setProgressValue(100); + gsv.setProgressText('Sync finished, redirecting...'); + window.location.href = gs.redirectUrl; +}); +gs.addHandler('error', function(data) { + progressTimers.forEach(function(timer) { clearInterval(timer); }); + gsv.setProgressValue(100); + gsv.setProgressText('Error: ' + data.message); + gsv.setProgressError(true); + gsv.setTerminalVisibility(true); + if (data.output) { + gsv.term.write(data.output); + } +}); +gs.start(); + +$('#header, #site').show(); + +// Make sure we provide plenty of appearances of progress! +var progressTimers = []; +progressTimers.push(setInterval(function() { + gsv.setProgressText(substatus_messages[Math.floor(Math.random() * substatus_messages.length)]); +}, 3000)); +progressTimers.push(setInterval(function() { + gsv.setProgressText(gsv.getProgressText() + '.'); +}, 800)); + +progressTimers.push(setInterval(function() { + // Illusion of progress! + gsv.setProgressValue(gsv.getProgressValue() + (0.01 * (100 - gsv.getProgressValue()))); +}, 900)); + + +var substatus_messages = [ + "Adding Hidden Agendas", + "Adjusting Bell Curves", + "Aesthesizing Industrial Areas", + "Aligning Covariance Matrices", + "Applying Feng Shui Shaders", + "Applying Theatre Soda Layer", + "Asserting Packed Exemplars", + "Attempting to Lock Back-Buffer", + "Binding Sapling Root System", + "Breeding Fauna", + "Building Data Trees", + "Bureacritizing Bureaucracies", + "Calculating Inverse Probability Matrices", + "Calculating Llama Expectoration Trajectory", + "Calibrating Blue Skies", + "Charging Ozone Layer", + "Coalescing Cloud Formations", + "Cohorting Exemplars", + "Collecting Meteor Particles", + "Compounding Inert Tessellations", + "Compressing Fish Files", + "Computing Optimal Bin Packing", + "Concatenating Sub-Contractors", + "Containing Existential Buffer", + "Debarking Ark Ramp", + "Debunching Unionized Commercial Services", + "Deciding What Message to Display Next", + "Decomposing Singular Values", + "Decrementing Tectonic Plates", + "Deleting Ferry Routes", + "Depixelating Inner Mountain Surface Back Faces", + "Depositing Slush Funds", + "Destabilizing Economic Indicators", + "Determining Width of Blast Fronts", + "Dicing Models", + "Diluting Livestock Nutrition Variables", + "Downloading Satellite Terrain Data", + "Eating Ice Cream", + "Exposing Flash Variables to Streak System", + "Extracting Resources", + "Factoring Pay Scale", + "Fixing Election Outcome Matrix", + "Flood-Filling Ground Water", + "Flushing Pipe Network", + "Gathering Particle Sources", + "Generating Jobs", + "Gesticulating Mimes", + "Graphing Whale Migration", + "Hiding Willio Webnet Mask", + "Implementing Impeachment Routine", + "Increasing Accuracy of RCI Simulators", + "Increasing Magmafacation", + "Initializing Rhinoceros Breeding Timetable", + "Initializing Robotic Click-Path AI", + "Inserting Sublimated Messages", + "Integrating Curves", + "Integrating Illumination Form Factors", + "Integrating Population Graphs", + "Iterating Cellular Automata", + "Lecturing Errant Subsystems", + "Modeling Object Components", + "Normalizing Power", + "Obfuscating Quigley Matrix", + "Overconstraining Dirty Industry Calculations", + "Partitioning City Grid Singularities", + "Perturbing Matrices", + "Polishing Water Highlights", + "Populating Lot Templates", + "Preparing Sprites for Random Walks", + "Prioritizing Landmarks", + "Projecting Law Enforcement Pastry Intake", + "Realigning Alternate Time Frames", + "Reconfiguring User Mental Processes", + "Relaxing Splines", + "Removing Road Network Speed Bumps", + "Removing Texture Gradients", + "Removing Vehicle Avoidance Behavior", + "Resolving GUID Conflict", + "Reticulating Splines", + "Retracting Phong Shader", + "Retrieving from Back Store", + "Reverse Engineering Image Consultant", + "Routing Neural Network Infanstructure", + "Scattering Rhino Food Sources", + "Scrubbing Terrain", + "Searching for Llamas", + "Seeding Architecture Simulation Parameters", + "Sequencing Particles", + "Setting Advisor Moods", + "Setting Inner Deity Indicators", + "Setting Universal Physical Constants", + "Smashing The Patriarchy", + "Sonically Enhancing Occupant-Free Timber", + "Speculating Stock Market Indices", + "Splatting Transforms", + "Stratifying Ground Layers", + "Sub-Sampling Water Data", + "Synthesizing Gravity", + "Synthesizing Wavelets", + "Time-Compressing Simulator Clock", + "Unable to Reveal Current Activity", + "Weathering Buildings", + "Zeroing Crime Network" +]; diff --git a/nbgitpuller/nbgitpuller/templates/page.html b/nbgitpuller/nbgitpuller/templates/page.html new file mode 100644 index 0000000000000000000000000000000000000000..aae54ad5dcf4e51ce6c0d423402b5b4fbd3b1031 --- /dev/null +++ b/nbgitpuller/nbgitpuller/templates/page.html @@ -0,0 +1,89 @@ + + + + + + {% block title %}Jupyter Server{% endblock %} + {% block favicon %}{% endblock %} + + + + + + + {% block stylesheet %} + {% endblock stylesheet %} + + {% block meta %} + {% endblock meta %} + + + + + + + + +
+ {% block site %} + {% endblock site %} +
+ + {% block after_site %} + {% endblock after_site %} + + {% block script %} + {% endblock script %} + + + + + \ No newline at end of file diff --git a/nbgitpuller/nbgitpuller/templates/status.html b/nbgitpuller/nbgitpuller/templates/status.html new file mode 100644 index 0000000000000000000000000000000000000000..f4388510f175c4be5db29a23877dd3e1092f85ae --- /dev/null +++ b/nbgitpuller/nbgitpuller/templates/status.html @@ -0,0 +1,75 @@ +{% extends "page.html" %} + +{% block params %} +{{super()}} +data-base-url="{{ base_url | urlencode }}" +data-repo="{{ repo | urlencode }}" +data-path="{{ path | urlencode }}" +{% if branch %}data-branch="{{ branch | urlencode }}"{% endif %} +{% if depth %}data-depth="{{ depth | urlencode }}"{% endif %} +data-targetpath="{{ targetpath | urlencode }}" +{% endblock %} + +{% block site %} +
+
+ Synchronizing git repository before sending you to {{ path }}... +
+ +
+
+
+
+ +
+
+ Click to see more details +
+ +
+
+{% endblock %} + +{% block script %} +{{super()}} + + +{% endblock %} + +{% block stylesheet %} +{{super()}} + + +{% endblock %} diff --git a/nbgitpuller/nbgitpuller/version.py b/nbgitpuller/nbgitpuller/version.py new file mode 100644 index 0000000000000000000000000000000000000000..fa57e1582d7e82a99469a66f24e4e2cd7dd66c65 --- /dev/null +++ b/nbgitpuller/nbgitpuller/version.py @@ -0,0 +1,2 @@ +""""The nbgitpuller PyPI package SemVer version.""" +__version__ = '0.10.2dev0' diff --git a/nbgitpuller/nbgitpuller/wget.py b/nbgitpuller/nbgitpuller/wget.py new file mode 100644 index 0000000000000000000000000000000000000000..652a0bdc3db37d19cd38d55b12b095d0bf3c97bf --- /dev/null +++ b/nbgitpuller/nbgitpuller/wget.py @@ -0,0 +1,61 @@ +import os +import io +import errno +import nest_asyncio +from tornado.httpclient import HTTPClient, HTTPRequest + +class RequestRepoRawFile: + + def wgetFile(self, repo, branch, path): + """ + Wget repository single file from code repository raw page + path: lab/tree/markdown-editor/code123/index.ipynb + """ + file_store_dir = 'tmp/' + namespace_path, repo_path = repo.rsplit('/', 1) + if path.startswith('lab/tree/'): + # lab + service = 'lab/tree/' + repo_file_path = path.replace('lab/tree/', '', 1) + repo_file_path = repo_file_path.replace(repo_path + '/', '', 1) + else: + # notebook + service = 'tree/' + repo_file_path = path.replace('tree/', '', 1) + repo_file_path = repo_file_path.replace(repo_path + '/', '', 1) + repo_file_store_path = file_store_dir + repo_path + '/' + repo_file_path + file_url = "%s/-/raw/%s/%s" % (repo, branch, repo_file_path) + repo_file_store_dir, file_name = repo_file_store_path.rsplit("/", 1) + + print(">", "mkdir " + repo_file_store_dir) + if not os.path.exists(repo_file_store_dir): + try: + os.makedirs(repo_file_store_dir, 0o755) + except OSError as e: + print(e) + if e.errno != errno.EEXIST: + raise + + print(">", "wget " + file_url) + file_content = self.wget(file_url) + file_content = file_content.decode('utf8', 'replace') + with io.open(repo_file_store_path, 'w', encoding='utf-8') as f: + f.write(file_content) + return service + repo_file_store_path + + def wget(self, url): + """ + get file content from a certain url + """ + nest_asyncio.apply() + req = HTTPRequest( + url, + method="GET", + headers={ + "Accept": "application/json", + "User-Agent": "CSDN JupyterHub" + } + ) + response = HTTPClient().fetch(req) + return response.body + diff --git a/nbgitpuller/package.json b/nbgitpuller/package.json new file mode 100644 index 0000000000000000000000000000000000000000..be972638eb20f92634e3aedc8e1e0c91c59598b5 --- /dev/null +++ b/nbgitpuller/package.json @@ -0,0 +1,28 @@ +{ + "name": "nbgitpuller", + "version": "0.10.1", + "description": "`nbgitpuller`", + "devDependencies": { + "jquery": "^3.6.0", + "webpack": "^5.45.1", + "webpack-cli": "^4.7.2", + "xterm": "^4.13.0", + "xterm-addon-fit": "^0.5.0", + "css-loader": "^6.2.0", + "style-loader": "^3.2.1" + }, + "scripts": { + "webpack": "webpack", + "webpack:watch": "webpack --watch" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/jupyterhub/nbgitpuller.git" + }, + "author": "", + "license": "BSD-3-Clause", + "bugs": { + "url": "https://github.com/jupyterhub/nbgitpuller/issues" + }, + "homepage": "https://github.com/jupyterhub/nbgitpuller#readme" +} diff --git a/nbgitpuller/setup.cfg b/nbgitpuller/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..5e4090017a9bec5de60a745d17007a9e95f31a43 --- /dev/null +++ b/nbgitpuller/setup.cfg @@ -0,0 +1,2 @@ +[wheel] +universal = 1 diff --git a/nbgitpuller/setup.py b/nbgitpuller/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..faea4fd0e34afa86eb9310e55a9fe4fd01bc5688 --- /dev/null +++ b/nbgitpuller/setup.py @@ -0,0 +1,48 @@ +from setuptools import find_packages, setup +from distutils.util import convert_path +import subprocess + +# Imports __version__, reference: https://stackoverflow.com/a/24517154/2220152 +ns = {} +ver_path = convert_path('nbgitpuller/version.py') +with open(ver_path) as ver_file: + exec(ver_file.read(), ns) +__version__ = ns['__version__'] + +subprocess.check_call(['npm', 'install']) +subprocess.check_call(['npm', 'run', 'webpack']) + +setup( + name='nbgitpuller', + version=__version__, + url='https://github.com/jupyterhub/nbgitpuller', + license='3-clause BSD', + author='Peter Veerman, YuviPanda', + author_email='peterkangveerman@gmail.com', + description='Notebook Extension to do one-way synchronization of git repositories', + long_description=open('README.md').read(), + long_description_content_type='text/markdown', + packages=find_packages(), + include_package_data=True, + platforms='any', + install_requires=['notebook>=5.5.0', 'jupyter_server>=1.10.1', 'tornado'], + data_files=[ + ('etc/jupyter/jupyter_server_config.d', ['nbgitpuller/etc/jupyter_server_config.d/nbgitpuller.json']), + ('etc/jupyter/jupyter_notebook_config.d', ['nbgitpuller/etc/jupyter_notebook_config.d/nbgitpuller.json']) + ], + zip_safe=False, + entry_points={ + 'console_scripts': [ + 'gitpuller = nbgitpuller.pull:main', + ], + }, + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: BSD License', + 'Operating System :: POSIX', + 'Operating System :: MacOS', + 'Operating System :: Unix', + 'Programming Language :: Python :: 3', + 'Topic :: Software Development :: Libraries :: Python Modules', + ] +) diff --git a/nbgitpuller/tests/test_api.py b/nbgitpuller/tests/test_api.py new file mode 100644 index 0000000000000000000000000000000000000000..4b062d14b11b3795d29beef6fea7fc68f3d7d953 --- /dev/null +++ b/nbgitpuller/tests/test_api.py @@ -0,0 +1,124 @@ +import os +from http.client import HTTPConnection +import subprocess +from time import sleep +from urllib.parse import quote +from uuid import uuid4 +import pytest + +PORT = os.getenv('TEST_PORT', 18888) + + +def request_api(params, host='localhost'): + h = HTTPConnection(host, PORT, 10) + query = '&'.join('{}={}'.format(k, quote(v)) for (k, v) in params.items()) + url = '/git-pull/api?token=secret&{}'.format(query) + h.request('GET', url) + return h.getresponse() + + +class TestNbGitPullerApi: + + def setup(self): + self.jupyter_proc = None + + def teardown(self): + if self.jupyter_proc: + self.jupyter_proc.kill() + + def start_jupyter(self, jupyterdir, extraenv, backend_type): + env = os.environ.copy() + env.update(extraenv) + if "server" in backend_type: + command = [ + 'jupyter-server', + '--NotebookApp.token=secret', + '--port={}'.format(PORT), + ] + else: + command = [ + 'jupyter-notebook', + '--no-browser', + '--NotebookApp.token=secret', + '--port={}'.format(PORT), + ] + self.jupyter_proc = subprocess.Popen(command, cwd=jupyterdir, env=env) + sleep(2) + + @pytest.mark.parametrize( + "backend_type", + [ + ("jupyter-server"), + ("jupyter-notebook"), + ], + ) + def test_clone_default(self, tmpdir, backend_type): + """ + Tests use of 'repo' and 'branch' parameters. + """ + jupyterdir = str(tmpdir) + self.start_jupyter(jupyterdir, {}, backend_type) + params = { + 'repo': 'https://github.com/binder-examples/jupyter-extension', + 'branch': 'master', + } + r = request_api(params) + assert r.code == 200 + s = r.read().decode() + print(s) + assert '--branch master' in s + assert "Cloning into '{}/{}'".format(jupyterdir, 'jupyter-extension') in s + assert os.path.isdir(os.path.join(jupyterdir, 'jupyter-extension', '.git')) + + @pytest.mark.parametrize( + "backend_type", + [ + ("jupyter-server"), + ("jupyter-notebook"), + ], + ) + def test_clone_targetpath(self, tmpdir, backend_type): + """ + Tests use of 'targetpath' parameter. + """ + jupyterdir = str(tmpdir) + target = str(uuid4()) + self.start_jupyter(jupyterdir, {}, backend_type) + params = { + 'repo': 'https://github.com/binder-examples/jupyter-extension', + 'branch': 'master', + 'targetpath': target, + } + r = request_api(params) + assert r.code == 200 + s = r.read().decode() + print(s) + assert "Cloning into '{}/{}'".format(jupyterdir, target) in s + assert os.path.isdir(os.path.join(jupyterdir, target, '.git')) + + @pytest.mark.parametrize( + "backend_type", + [ + ("jupyter-server"), + ("jupyter-notebook"), + ], + ) + def test_clone_parenttargetpath(self, tmpdir, backend_type): + """ + Tests use of the NBGITPULLER_PARENTPATH environment variable. + """ + jupyterdir = str(tmpdir) + parent = str(uuid4()) + target = str(uuid4()) + self.start_jupyter(jupyterdir, {'NBGITPULLER_PARENTPATH': parent}, backend_type) + params = { + 'repo': 'https://github.com/binder-examples/jupyter-extension', + 'branch': 'master', + 'targetpath': target, + } + r = request_api(params) + assert r.code == 200 + s = r.read().decode() + print(s) + assert "Cloning into '{}/{}/{}'".format(jupyterdir, parent, target) in s + assert os.path.isdir(os.path.join(jupyterdir, parent, target, '.git')) diff --git a/nbgitpuller/tests/test_gitpuller.py b/nbgitpuller/tests/test_gitpuller.py new file mode 100644 index 0000000000000000000000000000000000000000..0055b0dada49d4c6b1f8f46ee44bf222ede39d3a --- /dev/null +++ b/nbgitpuller/tests/test_gitpuller.py @@ -0,0 +1,429 @@ +import os +import shutil +import subprocess as sp +import glob +import time +import pytest + +from traitlets.config.configurable import Configurable + +from nbgitpuller import GitPuller + + +class Repository: + def __init__(self, path='remote'): + self.path = path + + def __enter__(self): + os.mkdir(self.path) + self.git('init', '--bare') + return self + + def __exit__(self, *args): + shutil.rmtree(self.path) + + def write_file(self, path, content): + with open(os.path.join(self.path, path), 'w') as f: + f.write(content) + + def read_file(self, path): + with open(os.path.join(self.path, path)) as f: + return f.read() + + def git(self, *args): + return sp.check_output( + ['git'] + list(args), + cwd=self.path, + stderr=sp.STDOUT + ).decode().strip() + + +class Remote(Repository): + pass + + +class Pusher(Repository): + def __init__(self, remote, path='pusher'): + self.remote = remote + super().__init__(path=path) + + def __enter__(self): + sp.check_output(['git', 'clone', self.remote.path, self.path]) + self.git('config', '--local', 'user.email', 'pusher@example.com') + self.git('config', '--local', 'user.name', 'pusher') + return self + + def push_file(self, path, content): + self.write_file(path, content) + self.git('add', path) + self.git('commit', '-am', 'Ignore the message') + self.git('push', 'origin', 'master') + + +class Puller(Repository): + def __init__(self, remote, path='puller', branch="master", *args, **kwargs): + super().__init__(path) + remotepath = "file://%s" % os.path.abspath(remote.path) + self.gp = GitPuller(remotepath, path, branch=branch, *args, **kwargs) + + def pull_all(self): + for line in self.gp.pull(): + print('{}: {}'.format(self.path, line.rstrip())) + + def __enter__(self): + print() + self.pull_all() + return self + + +# Tests to write: +# 1. Initialize puller with gitpuller, test for user config & commit presence +# 2. Push commit with pusher, pull with puller, valiate that nothing has changeed +# 3. Delete file in puller, run puller, make sure file is back +# 4. Make change in puller to file, make change in pusher to different part of file, run puller +# 5. Make change in puller to file, make change in pusher to same part of file, run puller +# 6. Make untracked file in puller, add file with same name to pusher, run puller + + +def test_initialize(): + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + + assert not os.path.exists('puller') + with Puller(remote, 'puller') as puller: + assert os.path.exists(os.path.join(puller.path, 'README.md')) + assert puller.git('name-rev', '--name-only', 'HEAD') == 'master' + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + + +def command_line_test_helper(remote_path, branch, pusher_path): + work_dir = "/".join(os.path.dirname(os.path.abspath(__file__)).split("/")[:-1]) + "/nbgitpuller" + try: + cmd = ['python3', 'pull.py', remote_path, branch, pusher_path] + sp.check_output( + cmd, + cwd=work_dir + ).decode() + return True + except Exception: + return False + + +def test_command_line_existing_branch(): + branch = "master" + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + remotepath = "file://%s" % os.path.abspath(remote.path) + pusherpath = os.path.abspath(pusher.path) + subprocess_result = command_line_test_helper(remotepath, branch, pusherpath) + assert subprocess_result + + +def test_command_line_default_branch(): + branch = "" + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + remotepath = "file://%s" % os.path.abspath(remote.path) + pusherpath = os.path.abspath(pusher.path) + subprocess_result = command_line_test_helper(remotepath, branch, pusherpath) + assert subprocess_result + + +def test_command_line_non_existing_branch(): + branch = "wrong" + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + remotepath = "file://%s" % os.path.abspath(remote.path) + pusherpath = os.path.abspath(pusher.path) + subprocess_result = command_line_test_helper(remotepath, branch, pusherpath) + assert not subprocess_result + + +def test_branch_exists(): + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + with Puller(remote, 'puller') as puller: + assert not puller.gp.branch_exists("wrong") + assert puller.gp.branch_exists("master") + + +def test_exception_branch_exists(): + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + with Puller(remote, 'puller') as puller: + orig_url = puller.gp.git_url + puller.gp.git_url = "" + try: + puller.gp.branch_exists("wrong") + except Exception as e: + assert type(e) == ValueError + puller.gp.git_url = orig_url + + +def test_resolve_default_branch(): + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + with Puller(remote, 'puller') as puller: + assert puller.gp.resolve_default_branch() == "master" + + +def test_exception_resolve_default_branch(): + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + with Puller(remote, 'puller') as puller: + orig_url = puller.gp.git_url + puller.gp.git_url = "" + try: + puller.gp.resolve_default_branch() + except Exception as e: + assert type(e) == ValueError + puller.gp.git_url = orig_url + + +def test_simple_push_pull(): + """ + Test the 'happy path' push/pull interaction + + 1. Push a file to remote, pull (initially) to make sure we get it + 2. Modify file & push to remote, pull to make sure we get update + 3. Add new file to remote, pull to make sure we get it + 4. Delete new file to remote, pull to make sure it is gone + + No modifications are done in the puller repo here, so we do not + exercise any merging behavior. + """ + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + + with Puller(remote) as puller: + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + assert puller.read_file('README.md') == pusher.read_file('README.md') == '1' + + pusher.push_file('README.md', '2') + puller.pull_all() + + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + assert puller.read_file('README.md') == pusher.read_file('README.md') == '2' + + pusher.push_file('another-file', '3') + + puller.pull_all() + + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + assert puller.read_file('another-file') == pusher.read_file('another-file') == '3' + + pusher.git('rm', 'another-file') + pusher.git('commit', '-m', 'Removing File') + pusher.git('push', 'origin', 'master') + + puller.pull_all() + + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + assert not os.path.exists(os.path.join(puller.path, 'another-file')) + + +def test_git_lock(): + """ + Test the 'happy path', but with stale/unstale git locks + """ + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + + with Puller(remote) as puller: + pusher.push_file('README.md', '2') + + puller.write_file('.git/index.lock', '') + + exception_raised = False + try: + puller.pull_all() + except Exception: + exception_raised = True + assert exception_raised + + new_time = time.time() - 700 + os.utime(os.path.join(puller.path, '.git', 'index.lock'), (new_time, new_time)) + + puller.pull_all() + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + + +def test_merging_simple(): + """ + Test that when we change local & remote, local changes are preferred + """ + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + + with Puller(remote) as puller: + assert puller.read_file('README.md') == pusher.read_file('README.md') == '1' + + puller.write_file('README.md', '2') + + pusher.push_file('README.md', '3') + + puller.pull_all() + + # There should be a commit made *before* the pull that has our explicit + # authorship, to record that it was made by nbgitpuller + assert puller.git('show', '-s', '--format="%an <%ae>"', 'HEAD^1') == '"nbgitpuller "' + + assert puller.read_file('README.md') == '2' + assert pusher.read_file('README.md') == '3' + + # Make sure that further pushes to other files are reflected + pusher.push_file('another-file', '4') + + puller.pull_all() + + assert puller.read_file('another-file') == pusher.read_file('another-file') == '4' + + # Make sure our merging works across commits + + pusher.push_file('README.md', '5') + puller.pull_all() + + assert puller.read_file('README.md') == '2' + + +def test_untracked_puller(): + """ + Test that untracked files in puller are preserved when pulling + """ + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + + with Puller(remote) as puller: + pusher.push_file('another-file', '2') + + puller.write_file('another-file', '3') + + puller.pull_all() + assert puller.read_file('another-file') == '2' + # Find file that was created! + renamed_file = glob.glob(os.path.join(puller.path, 'another-file_*'))[0] + assert puller.read_file(os.path.basename(renamed_file)) == '3' + + +def test_reset_file(): + """ + Test that deleting files locally & pulling restores pristine copy + """ + with Remote() as remote, Pusher(remote) as pusher: + pusher.push_file('README.md', '1') + pusher.push_file('unicode🙂.txt', '2') + + with Puller(remote) as puller: + os.remove(os.path.join(puller.path, 'README.md')) + os.remove(os.path.join(puller.path, 'unicode🙂.txt')) + + puller.pull_all() + + assert puller.git('rev-parse', 'HEAD') == pusher.git('rev-parse', 'HEAD') + assert puller.read_file('README.md') == pusher.read_file('README.md') == '1' + assert puller.read_file('unicode🙂.txt') == pusher.read_file('unicode🙂.txt') == '2' + + +@pytest.fixture(scope='module') +def long_remote(): + with Remote("long_remote") as remote, Pusher(remote, "lr_pusher") as pusher: + for i in range(0, 10): + pusher.git('commit', '--allow-empty', '-m', "Empty message %d" % i) + pusher.git('push', 'origin', 'master') + + yield remote + + +@pytest.fixture(scope="function") +def clean_environment(): + """ + Save and restore the state of named VARIABLES before, during, and + after tests. + """ + + VARIABLES = ['NBGITPULLER_DEPTH'] + backups = {} + for var in VARIABLES: + backups[var] = os.environ.get(var) + if backups[var]: + del os.environ[var] + + yield + + for var in backups: + if backups[var]: + os.environ[var] = backups[var] + elif os.environ.get(var): + del os.environ[var] + + +def count_loglines(repository): + return len(repository.git('log', '--oneline').split("\n")) + + +def test_unshallow_clone(long_remote, clean_environment): + """ + Sanity-test that clones with 10 commits have 10 log entries + """ + os.environ['NBGITPULLER_DEPTH'] = "0" + with Puller(long_remote, 'normal') as puller: + assert count_loglines(puller) == 10 + + +def test_shallow_clone(long_remote, clean_environment): + """ + Test that shallow clones only have a portion of the git history + """ + with Puller(long_remote, 'shallow4', depth=4) as puller: + assert count_loglines(puller) == 4 + + +def test_shallow_clone_config(long_remote, clean_environment): + """ + Test that shallow clones can be configured via parent Configurables + """ + class TempConfig(Configurable): + def __init__(self): + super(TempConfig) + self.config['GitPuller']['depth'] = 5 + + with Puller(long_remote, 'shallow4', parent=TempConfig()) as puller: + assert count_loglines(puller) == 5 + + +def test_environment_shallow_clone(long_remote, clean_environment): + """ + Test that shallow clones respect the NBGITPULLER_DEPTH environment variable + by default + """ + os.environ['NBGITPULLER_DEPTH'] = "2" + with Puller(long_remote, 'shallow_env') as puller: + assert count_loglines(puller) == 2 + + +def test_explicit_unshallow(long_remote, clean_environment): + """ + Test that we can disable environment-specified shallow clones + """ + os.environ['NBGITPULLER_DEPTH'] = "2" + with Puller(long_remote, 'explicitly_full', depth=0) as puller: + assert count_loglines(puller) == 10 + + +def test_pull_on_shallow_clone(long_remote, clean_environment): + """ + Test that we can perform a pull on a shallow clone + """ + with Puller(long_remote, depth=0) as shallow_puller: + with Pusher(long_remote) as pusher: + pusher.push_file('test_file', 'test') + + orig_head = shallow_puller.git('rev-parse', 'HEAD') + shallow_puller.pull_all() + new_head = shallow_puller.git('rev-parse', 'HEAD') + upstream_head = long_remote.git('rev-parse', 'HEAD') + + assert orig_head != new_head + assert new_head == upstream_head + + pusher.git('push', '--force', 'origin', '%s:master' % orig_head) diff --git a/nbgitpuller/webpack.config.js b/nbgitpuller/webpack.config.js new file mode 100644 index 0000000000000000000000000000000000000000..0b769d5f96ef08d2c5ace95cb0bef3a7388c5c38 --- /dev/null +++ b/nbgitpuller/webpack.config.js @@ -0,0 +1,26 @@ +const webpack = require('webpack'); + +module.exports = { + context: __dirname + "/nbgitpuller/static/", + entry: "./js/index.js", + output: { + path: __dirname + "/nbgitpuller/static/dist/", + filename: "bundle.js", + publicPath: '/static/dist/' + }, + module: { + rules: [ + { + test: /\.css$/i, + use: ['style-loader', 'css-loader'] + }, + ] + }, + devtool: 'source-map', + plugins: [ + new webpack.ProvidePlugin({ + $: 'jquery', + jQuery: 'jquery', + }), + ] +} diff --git a/requirements.txt b/requirements.txt index add4b6f2690f10419c901dfb7326901f801c2ceb..9105393a67f3c0ae0ba9f36e6d468d375570aaed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ jupyterhub==1.4.2 -nbgitpuller==0.10.1 \ No newline at end of file +# nbgitpuller==0.10.1 \ No newline at end of file