Remove dependence on repo2docker (#91)

* Changes for simple container

Makefile changes:
  - 'build' target now builds simple container
  - 'build-old' target builds the old repo2docker way
  - 'debug-build' for docker build issues
  - docker image tag includes git ref & date stamp

Dockerfile changes:
    - pass hadolint
    - change permissions to allow notebook signing for trust
    - update user name for new base image

set_secrets_in_env.sh changes:
    - support '--develop' option to set bogus values for testing jupyter
      & notebook parts

* Update Notebook so signing works

Somewhere along the line, jupyter changed to needing an 'id' property
per cell for proper signing, without changing the ipynb version number.

Fix is to use `nbconvert --to ipynb` to rewrite file.

* Build Environment Hygene

- Update versions of pre-commit
  - updated hadolint config
- Make things more normal
  - stop using port 10001
  - add '--unset' option for testing
- Cleaned up Dockerfile
- Cleaned up Makefile
  - removed method for older image build method
- updated documenation for new image build process
- removed no-longer-needed binder config files

* Fix bug with eliminating first name from ACL search

Co-authored-by: Hal Wine <[email protected]>

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.2.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -15,18 +15,18 @@ repos:
     -   id: check-toml
 
 -   repo: https://github.com/psf/black
-    rev: 21.4b2
+    rev: 22.3.0
     hooks:
     -   id: black
 
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.14.0
+    rev: v2.32.1
     hooks:
     -   id: pyupgrade
         args: ["--py37-plus"]
 
 - repo: https://github.com/jumanjihouse/pre-commit-hooks
-  rev: 2.1.5  # or specific git tag
+  rev: 2.1.6  # or specific git tag
   hooks:
     - id: forbid-binary
       #    - id: markdownlint # Configure in .mdlrc
@@ -38,13 +38,17 @@ repos:
       ]
 
 - repo: https://github.com/IamTheFij/docker-pre-commit
-  rev: v2.0.0
+  rev: v2.1.0
   hooks:
         - id: docker-compose-check
-        - id: hadolint
+
+- repo: https://github.com/hadolint/hadolint
+  rev: v2.10.0
+  hooks:
+    - id: hadolint
 
 - repo: https://github.com/PyCQA/bandit
-  rev: 1.7.0
+  rev: 1.7.4
   hooks:
     - id: bandit
 
@@ -62,7 +66,7 @@ repos:
     #args: [--meta, ExecuteTime]  # Optional blacklist of metadata keys (you can use regex)
 
 - repo: https://github.com/Yelp/detect-secrets
-  rev: v1.1.0
+  rev: v1.2.0
   hooks:
     - id: detect-secrets
       args: ['--baseline', '.secrets.baseline']

.secrets.baseline

@@ -1,5 +1,5 @@
 {
-  "version": "1.1.0",
+  "version": "1.2.0",
   "plugins_used": [
     {
       "name": "ArtifactoryDetector"
@@ -20,6 +20,9 @@
     {
       "name": "CloudantDetector"
     },
+    {
+      "name": "GitHubTokenDetector"
+    },
     {
       "name": "HexHighEntropyString",
       "limit": 3
@@ -46,6 +49,9 @@
     {
       "name": "PrivateKeyDetector"
     },
+    {
+      "name": "SendGridDetector"
+    },
     {
       "name": "SlackDetector"
     },
@@ -107,9 +113,9 @@
       {
         "type": "Hex High Entropy String",
         "filename": "notebooks/UserSearchPy3.ipynb",
-        "hashed_secret": "f93224b55c82d1c2b92c45e88d9c96d368d34405",
+        "hashed_secret": "13b897fb3181b06360814e15a2535df2624de13a",
         "is_verified": false,
-        "line_number": 2006,
+        "line_number": 1928,
         "is_secret": false
       }
     ],
@@ -124,5 +130,5 @@
       }
     ]
   },
-  "generated_at": "2022-03-06T17:49:52Z"
+  "generated_at": "2022-05-14T02:05:41Z"
 }

Dockerfile

@@ -0,0 +1,25 @@
+FROM jupyter/minimal-notebook:hub-2.1.1
+
+# Define the subdirectory with notebooks
+ARG NOTEBOOK_DIR="notebooks"
+
+# add in the extensions we need
+RUN    conda install -y -c conda-forge jupyter_contrib_nbextensions ; \
+       jupyter nbextensions_configurator enable ; \
+       jupyter nbextension enable init_cell/main ; \
+       jupyter nbextension enable collapsible_headings/main ; \
+       jupyter nbextension list
+
+# install the packages needed
+WORKDIR /home/jovyan
+COPY    --chown=jovyan:users requirements.txt .
+RUN     pip install --no-cache-dir -r requirements.txt && \
+        rmdir work && \
+        rm requirements*.txt
+
+USER jovyan
+# install the notebooks and trust the notebooks we ship
+COPY --chown=jovyan:users notebooks/*.ipynb ${NOTEBOOK_DIR}/
+RUN    echo "pwd $PWD" ; \
+       echo "ls: $(ls -lAd ./* ${NOTEBOOK_DIR}/*.ipynb)" ; \
+       jupyter trust ${NOTEBOOK_DIR}/*.ipynb

Makefile

@@ -1,8 +1,10 @@
-VENV_NAME:=venv
-github3_version:=1.1.0
-port := 10001
-image_to_use := offboard-py3
-container_user_name := ghjupyter
+gitrev := $(shell git rev-parse --short=10 HEAD)
+VENV_NAME := venv
+now := $(shell date --utc +%Y%m%dT%H%MZ)
+github3_version:=1.1.0-$(now)-$(gitrev)
+port := 8888
+image_to_use := offboard-slim
+container_user_name := jovyan
 SOPS_credentials := $(SECOPS_SOPS_PATH)/off-boarding.yaml
 
 DOCKER_OPTS :=
@@ -15,30 +17,23 @@ help:
 	@echo "    build         create a docker image based on working directory"
 	@echo "    run           run a docker image previously created"
 	@echo "    run-edit      run with modifiable current directory"
-	@echo "    jupyter 	 run local (non docker) jupyter server for development"
-	@echo "    $(VENV_NAME)  create a local virtualenv for old style development"
+	@echo "    jupyter       run local (non docker) jupyter server for development"
+	@echo "    $(VENV_NAME)          create a local virtualenv for old style development"
 
 $(VENV_NAME):
-	virtualenv --python=python3.7 $@
+	python3 -m venv $@
 	. $(VENV_NAME)/bin/activate && echo req*.txt | xargs -n1 pip install -r
 	@echo "Virtualenv created in $(VENV_NAME). You must activate before continuing."
 	false
 
 SHELL := /bin/bash
-.PHONY: build
-# we use a file url to avoid including work files in the production
-# image. During development, you may prefer a bare dot "." to pick up
-# local changes, and remove the `--ref ` option
-build:
-	-docker tag $(image_to_use):$(github3_version) $(image_to_use):$(github3_version)-previous 2>/dev/null
-	$(SHELL) -c '  \
-		repo2docker --image-name "$(image_to_use):$(github3_version)" \
-			--user-name $(container_user_name) \
-			--no-run \
-			--ref $$(git show-ref --verify --hash --head HEAD) \
-			file://$$PWD/.git \
-		; \
-	'
+.PHONY: build debug_build
+# New build
+build: Dockerfile .dockerignore Makefile notebooks/*ipynb requirements*.txt
+	docker build --tag $(image_to_use):$(github3_version) --tag $(image_to_use):latest .
+# debug the build by not using buildkit - we also assume last one failed, so no need to tag prior
+debug-build:
+	DOCKER_BUILDKIT=0 docker build --tag $(image_to_use):debug .
 
 # For `run`, we use the configs baked into the image at the time of
 # the build, so we get what we expect.
@@ -51,17 +46,19 @@ run:
 			--env "CIS_CLIENT_ID" \
 			--env "CIS_CLIENT_SECRET" \
 			--env "TZ" \
+			--env "DOCKER_STACKS_JUPYTER_CMD=notebook" \
 			--publish $(port):8888 \
-			$(image_to_use):$(github3_version) \
+			$(image_to_use):latest \
 		& \
 		job_pid=$$! ; \
 		sleep 5 ; \
-		docker ps --filter "ancestor=$(image_to_use):$(github3_version)" ; \
+		docker ps --filter "ancestor=$(image_to_use)" ; \
 		wait $$job_pid ; \
 	) '
 
-# For `run-edit`, we're mapping the current directory atop the home
-# directory
+# For `run-edit`, we're mapping the local notebooks directory onto the container's notebooks directory
+# This allows for editing the notebook, but still uses the baked in jupyter configuration (in $HOME)
+
 .PHONY: run-edit
 run-edit:
 	$(SHELL) -ce ' ( source set_secrets_in_env.sh $(SOPS_credentials); \
@@ -72,12 +69,13 @@ run-edit:
 			--env "CIS_CLIENT_ID" \
 			--env "CIS_CLIENT_SECRET" \
 			--env "TZ" \
+			--env "DOCKER_STACKS_JUPYTER_CMD=notebook" \
 			--publish $(port):8888 \
-			--volume "$$PWD/notebooks:/home/$(container_user_name)/notebooks" \
-			$(image_to_use):$(github3_version) \
+			--volume "$$PWD/notebooks:/home/$(container_user_name)"/notebooks \
+			$(image_to_use):latest \
 		& \
 		job_pid=$$! ; \
-		sleep 5 ; \
+		sleep 10 ; \
 		docker ps --filter "ancestor=$(image_to_use):$(github3_version)" ; \
 		wait $$job_pid ; \
 	) '

README.md

@@ -12,7 +12,7 @@ Supplying credentials for execution is done by passing a PAT token as the value
 of the environment variable `GITHUB_TOKEN` (preferred) or `GITHUB_PAT`.
 
 The recommended way to set `GITHUB_TOKEN` is via cli access to your password
-manager. For example, using [pass][pass]:
+manager. For example, using [pass]:
 ```bash
 GITHUB_TOKEN=$(pass show myPAT) script args
 ```
@@ -21,22 +21,19 @@ GITHUB_TOKEN=$(pass show myPAT) script args
 ## Jupyter Notebooks
 ### Docker Images
 
-The Jupyter Notebooks has a complex environment as regards dependencies. The recommended way to
-deal with this is by using a docker container. Given the complexities of setting up Jupyter
-to run in docker, a helper utility is use: `repo2docker`. The make targets
-assume that is installed. Use `pipx install jupyter-repo2docker` to install.
-_(See `README.md` files in the `binder/` directory tree for more info on building the image)_
+Our Jupyter Notebooks have a farely simple environment as regards dependencies. The recommended way to
+deal with this is by using a docker container.
 
 The Makefile contains targets for building and running the docker images. Invoke
 `make` without arguments to see those targets
 
-- **NOTE**: the docker image allows credentials to be supplied via [sops][sops].
+- **NOTE**: the docker image allows credentials to be supplied via [sops].
   The environment variable "`SECOPS_SOPS_PATH`" must be set appropriately.
 
 [sops]: https://github.com/mozilla/sops
 
 When started, the docker container will serve notebooks from the `notebooks/`
-directory. Current notebooks include:
+directory, but they will be available at the top level. Current notebooks include:
 
 - **`User Search.ipynb`** --
     Given a set of possible GitHub logins, determine if they might have any
@@ -55,7 +52,7 @@ encounter.
 
 ### auditlog.py
 Download audit log for $ORG via headless firefox via selenium
-([``geckodriver``][gd_url] must be installed). Credentials as environment
+([`geckodriver`][gd_url] must be installed). Credentials as environment
 variables, and 2FA token passed as input when requested.
 
 ### contributing.py