rtfin2022 release: template, revamped readme, docker updates (#100)

Major changes: - Separate instructions for Docker vs local conda environment - Installation instructions and template project tutorial linked together to demonstrate RT-Cloud usage - Install instructions now correctly account for different operating systems, with caveats for Mac M1s + Windows - Components have been renamed (data_analyser, data_streamer, analysis_listener) and the corresponding shell scripts have been renamed (but the original files are untouched so previous users do not need to modify anything, the renamed shell scripts are identical to the original shell scripts aside from the filename) - New picture depicting the communication of RT-Cloud components in Readme - Quickstart Docker demo - setResultsDict now conveniently outputs jsons to a default folder (this is demonstrated in the template project) - More specific instructions now provided in the run-in-docker markdown file - Replaying OpenNeuro section directs user to try to implement changes to the template project which is built to accommodate such Replay - New section: Sharing your RT-Cloud pipeline as a Docker image
brainiak · Oct 12, 2022 · cb98615 · cb98615
1 parent f2b4197
commit cb98615
Show file tree

Hide file tree

Showing 28 changed files with 776 additions and 324 deletions.
diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml
@@ -39,7 +39,7 @@ jobs:
 
     - name: Install bids-validator for test suite
       run: |
-        sudo npm install -g bids-validator
+        sudo npm install -g bids-validator@1.7.3
 
     - name: Install conda environment
       run: |
@@ -62,3 +62,4 @@ jobs:
     - name: Test with pytest
       run: |
         python -m pytest tests/
+        
diff --git a/Readme.md b/Readme.md
diff --git a/docker/Dockerfile.rtcloud b/docker/Dockerfile.rtcloud
@@ -1,23 +1,18 @@
-FROM brainiak/rtconda:1.3 as CONDA_IMAGE
-FROM brainiak/rttools:1.3
+FROM brainiak/rttools:1.4
 
 SHELL ["/bin/bash", "-c"]
 
-# Copy miniconda environment from the conda_image
-COPY --from=CONDA_IMAGE /opt/miniconda3 /opt/miniconda3
-COPY --from=CONDA_IMAGE /opt/nodejs /opt/nodejs
-COPY --from=CONDA_IMAGE /root/.bashrc /tmp/conda_bashrc
-
 # Get latests commit tag - to prevent caching old git clone of repo
 ADD https://api.github.com/repos/brainiak/rt-cloud/git/refs/heads/master version.json
 
 # Combine bashrc files from the conda_image
-WORKDIR /root
+WORKDIR /
 RUN \
+# Activate conda and bashrc 
+source ~/.bashrc && \
+conda activate rtcloud && \
+# Install rt-cloud from github
 mkdir /rt-cloud && \
-cp .bashrc bashrc.bak && \
-grep -Fvxf .bashrc /tmp/conda_bashrc > /tmp/bashrc_patch && \
-cat bashrc.bak /tmp/bashrc_patch > .bashrc && \
 echo "## Clone rt-cloud repo ##" && \
 git clone https://github.com/brainiak/rt-cloud.git /rt-cloud && \
 mkdir /rt-cloud/certs && \
@@ -27,12 +22,10 @@ source ~/.bashrc && \
 cd /rt-cloud/web && \
 npm install && \
 npm run build && \
-rm -r /rt-cloud/.git && \
 echo "## Completed Installing rt-cloud ##" && \
-# Copy things out of $HOME for use by singularity
+# Copy .bashrc out of $HOME for use by singularity
 cp ~/.bashrc /opt/.bashrc && \
-cp -r ~/.vnc /opt/.vnc && \
-echo "## Copied .bashrc and /.vnc for Singularity ##"
+echo "## Copied .bashrc for Singularity ##"
 
 EXPOSE 8888
 WORKDIR /rt-cloud
@@ -45,4 +38,4 @@ SHELL ["/bin/bash", "-ci"]
 CMD ["scripts/run-projectInterface.sh", "--test", "-p", "sample"]
 
 # Bash in interactive mode is entrypoint
-ENTRYPOINT ["/bin/bash", "-ci", "$0 $@"]
+ENTRYPOINT ["/bin/bash", "-ci", "$0 $@"]
diff --git a/docker/Dockerfile.rtcloudXL b/docker/Dockerfile.rtcloudXL
@@ -1,23 +1,18 @@
-FROM brainiak/rtconda:1.3 as CONDA_IMAGE
-FROM brainiak/rttoolsxl:1.3
+FROM brainiak/rttoolsxl:1.4
 
 SHELL ["/bin/bash", "-c"]
 
-# Copy miniconda environment from the conda_image
-COPY --from=CONDA_IMAGE /opt/miniconda3 /opt/miniconda3
-COPY --from=CONDA_IMAGE /opt/nodejs /opt/nodejs
-COPY --from=CONDA_IMAGE /root/.bashrc /tmp/conda_bashrc
-
 # Get latests commit tag - to prevent caching old git clone of repo
 ADD https://api.github.com/repos/brainiak/rt-cloud/git/refs/heads/master version.json
 
 # Combine bashrc files from the conda_image
-WORKDIR /root
+WORKDIR /
 RUN \
+# Activate conda and bashrc 
+source ~/.bashrc && \
+conda activate rtcloud && \
+# Install rt-cloud from github
 mkdir /rt-cloud && \
-cp .bashrc bashrc.bak && \
-grep -Fvxf .bashrc /tmp/conda_bashrc > /tmp/bashrc_patch && \
-cat bashrc.bak /tmp/bashrc_patch > .bashrc && \
 echo "## Clone rt-cloud repo ##" && \
 git clone https://github.com/brainiak/rt-cloud.git /rt-cloud && \
 mkdir /rt-cloud/certs && \
@@ -27,12 +22,10 @@ source ~/.bashrc && \
 cd /rt-cloud/web && \
 npm install && \
 npm run build && \
-rm -r /rt-cloud/.git && \
 echo "## Completed Installing rt-cloud ##" && \
-# Copy things out of $HOME for use by singularity
+# Copy .bashrc out of $HOME for use by singularity
 cp ~/.bashrc /opt/.bashrc && \
-cp -r ~/.vnc /opt/.vnc && \
-echo "## Copied .bashrc and /.vnc for Singularity ##"
+echo "## Copied .bashrc for Singularity ##"
 
 EXPOSE 8888
 WORKDIR /rt-cloud
@@ -45,4 +38,4 @@ SHELL ["/bin/bash", "-ci"]
 CMD ["scripts/run-projectInterface.sh", "--test", "-p", "sample"]
 
 # Bash in interactive mode is entrypoint
-ENTRYPOINT ["/bin/bash", "-ci", "$0 $@"]
+ENTRYPOINT ["/bin/bash", "-ci", "$0 $@"]
diff --git a/docker/Dockerfile.rtcloudlite b/docker/Dockerfile.rtcloudlite
@@ -1,23 +1,18 @@
-FROM brainiak/rtconda:1.3 as CONDA_IMAGE
-FROM brainiak/rttoolslite:1.3
+FROM brainiak/rttoolslite:1.4
 
 SHELL ["/bin/bash", "-c"]
 
-# Copy miniconda environment from the conda_image
-COPY --from=CONDA_IMAGE /opt/miniconda3 /opt/miniconda3
-COPY --from=CONDA_IMAGE /opt/nodejs /opt/nodejs
-COPY --from=CONDA_IMAGE /root/.bashrc /tmp/conda_bashrc
-
 # Get latests commit tag - to prevent caching old git clone of repo
 ADD https://api.github.com/repos/brainiak/rt-cloud/git/refs/heads/master version.json
 
 # Combine bashrc files from the conda_image
-WORKDIR /root
+WORKDIR /
 RUN \
+# Activate conda and bashrc 
+source ~/.bashrc && \
+conda activate rtcloud && \
+# Install rt-cloud from github
 mkdir /rt-cloud && \
-cp .bashrc bashrc.bak && \
-grep -Fvxf .bashrc /tmp/conda_bashrc > /tmp/bashrc_patch && \
-cat bashrc.bak /tmp/bashrc_patch > .bashrc && \
 echo "## Clone rt-cloud repo ##" && \
 git clone https://github.com/brainiak/rt-cloud.git /rt-cloud && \
 mkdir /rt-cloud/certs && \
@@ -27,12 +22,10 @@ source ~/.bashrc && \
 cd /rt-cloud/web && \
 npm install && \
 npm run build && \
-rm -r /rt-cloud/.git && \
 echo "## Completed Installing rt-cloud ##" && \
-# Copy things out of $HOME for use by singularity
+# Copy .bashrc out of $HOME for use by singularity
 cp ~/.bashrc /opt/.bashrc && \
-cp -r ~/.vnc /opt/.vnc && \
-echo "## Copied .bashrc and /.vnc for Singularity ##"
+echo "## Copied .bashrc for Singularity ##"
 
 EXPOSE 8888
 WORKDIR /rt-cloud
@@ -45,4 +38,4 @@ SHELL ["/bin/bash", "-ci"]
 CMD ["scripts/run-projectInterface.sh", "--test", "-p", "sample"]
 
 # Bash in interactive mode is entrypoint
-ENTRYPOINT ["/bin/bash", "-ci", "$0 $@"]
+ENTRYPOINT ["/bin/bash", "-ci", "$0 $@"]
diff --git a/docker/Dockerfile.rtconda b/docker/Dockerfile.rtconda
@@ -1,48 +1,45 @@
-FROM insgeek/centos:7.5
+FROM centos:7.5.1804
 
 SHELL ["/bin/bash", "-c"]
 
-# Install Conda
 RUN \
-yum install -y git wget ca-certificates unzip && \
-mkdir ~/Downloads && \
+yum install -y git wget ca-certificates && \
+mkdir /opt/Downloads && \
+echo "## Get rtcloud environments setup ##" && \
+wget https://raw.githubusercontent.com/brainiak/rt-cloud/master/environment.yml -P /tmp/ && \
+wget https://raw.githubusercontent.com/brainiak/rt-cloud/master/environment-synthetic-data.yml -P /tmp/ && \
+wget https://raw.githubusercontent.com/brainiak/rt-cloud/master/websockify.yml -P /tmp/ && \
 echo "## Install Miniconda ##" && \
-pushd ~/Downloads && \
+pushd /opt/Downloads && \
 wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
 bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda3 && \
 echo export PATH="/opt/miniconda3/bin:\$PATH" >> ~/.bashrc && \
 echo ". /opt/miniconda3/etc/profile.d/conda.sh" >> ~/.bashrc && \
 source ~/.bashrc && \
 conda update -y conda && \
-rm Miniconda3-latest-Linux-x86_64.sh && \
+popd && \
+rm -r /opt/Downloads && \
 echo "## Install conda environment ##" && \
 conda create --name rtcloud && \
 conda activate rtcloud && \
-conda install -y -c defaults -c conda-forge awscli bcrypt boto3 dcm2niix flake8 indexed_gzip jupyter mypy nibabel nilearn pip pydicom python=3.7 pytest requests rpyc scipy toml websocket-client wsaccel && \
+conda install -y -c defaults -c conda-forge awscli bcrypt boto3 dcm2niix flake8 indexed_gzip jupyter mypy nibabel nilearn nodejs pip pydicom python=3.7 requests rpyc scipy toml tornado websocket-client wsaccel && \
 pip install inotify pybids watchdog && \
-# nodejs refuses to install a version greater than 12.12, which is required for bids-validator, so need to install separately
-wget https://nodejs.org/dist/v16.15.0/node-v16.15.0-linux-x64.tar.xz && \
-xz -d node-v16.15.0-linux-x64.tar.xz && \
-tar xvf node-v16.15.0-linux-x64.tar && \
-mv node-v16.15.0-linux-x64 /opt/nodejs && \
-echo export PATH="/opt/nodejs/bin:\$PATH" >> ~/.bashrc && \
-popd && \
 echo "## Install synthetic-data environment ##" && \
 conda install -y -c brainiak -c defaults -c conda-forge brainiak && \
 conda install -y -c defaults -c conda-forge nomkl && \
-conda deactivate && \
+yes | conda clean --all && \
 echo "## Install VNC environment ##" && \
 yum install -y openssl tk && \
+conda deactivate && \
 conda create --name websockify && \
 conda activate websockify && \
 conda install -y -c defaults ca-certificates certifi libedit libffi libgcc-ng libstdcxx-ng ncurses openssl pip python readline setuptools sqlite tk tornado wheel xz zlib && \
 pip install numpy websockify && \
-conda deactivate && \
 echo "## Conda Install Complete ##" && \
 # Cleanup 
-rm -r ~/Downloads && \
 yum clean all && \
-conda clean --all && \
+yes | conda clean --all && \
+conda deactivate && \
 echo "## Cleanup complete ##" 
 
 ENV PATH="/opt/miniconda3/bin:${PATH}"

diff --git a/docker/Dockerfile.rttools b/docker/Dockerfile.rttools
@@ -1,17 +1,16 @@
-FROM centos:7.5.1804
+FROM brainiak/rtconda:1.4 
 
 RUN \
-# Yum install packages
-yum install -y git wget ca-certificates epel-release && \
-# Packages needed for FSL install (note: "openblas-devel.x86_64" requires "epel-release" be installed)
-yum -y install file which zip gzip bzip2 libpng12 libmng mesa-libGL-devel openblas-devel.x86_64 && \
-# Install Neuroscience Tools
-mkdir ~/Downloads && \
+# Packages needed for FSL install
+yum install -y file which zip gzip bzip2 libpng12 libmng mesa-libGL-devel epel-release && \
+# Activate conda and bashrc 
+source ~/.bashrc && \
+conda activate rtcloud && \
+yum -y install openblas-devel.x86_64 && \
 # Install FSL
-echo "## Install FSL ##"  && \
-pushd ~/Downloads && \
+cd /opt && \
 wget https://fsl.fmrib.ox.ac.uk/fsldownloads/fslinstaller.py && \
-/usr/bin/python2.7 fslinstaller.py -d /opt/fsl --quiet && \
+/usr/bin/python2.7 fslinstaller.py -d /opt/fsl -B centos7_64 && \
 echo 'FSLDIR=/opt/fsl' >> ~/.bashrc && \
 echo '. ${FSLDIR}/etc/fslconf/fsl.sh' >> ~/.bashrc && \
 echo 'PATH=${FSLDIR}/bin:${PATH}' >> ~/.bashrc && \
@@ -22,22 +21,23 @@ echo "## FSL Install Complete ##" && \
 echo "## Removing non-essential FSL utilites! ##" && \
 source ~/.bashrc && \
 cd $FSLDIR && \
-rm -r config data doc extras fslpython include lib python refdoc src tcl  && \
-rm -r etc/default_flobs.flobs etc/flirtsch etc/js etc/luts etc/matlab && \
-rm build build.log && \
-rm ${FSLDIR}/bin/fslview* ${FSLDIR}/bin/eddy* && \
+yes | rm -r config data doc extras fslpython include lib python refdoc src tcl && \
+yes | rm -r etc/default_flobs.flobs etc/flirtsch etc/js etc/luts etc/matlab && \
+yes | rm build build.log && \
+yes | rm ${FSLDIR}/bin/fslview* ${FSLDIR}/bin/eddy* && \
+cd / && \
 echo "## Unbulking FSL Complete ##" && \
 # Install ANTs (reduced version with limited functions)
 echo "## Install ANTs ##" && \
-cd ~/Downloads && \
 wget https://sourceforge.net/projects/advants/files/ANTS/ANTS_Latest/ANTs-2.1.0-rc3-Linux.tar.gz/download -O ANTs-2.1.0-rc3-Linux.tar.gz && \
 tar xzvf ANTs-2.1.0-rc3-Linux.tar.gz && \
 mv ANTs-2.1.0-Linux /opt/ && \
 cd /opt/ANTs-2.1.0-Linux/bin && \
-rm -r /opt/ANTs-2.1.0-Linux/lib && \
+yes | rm -r /opt/ANTs-2.1.0-Linux/lib && \
 ls -1 | egrep -v "^(ANTS|antsApplyTransforms|antsApplyTransformsToPoints|antsAlignOrigin|antsRegistration|antsTransformInfo)$" | xargs rm -r && \
 echo 'export PATH=${PATH}:/opt/ANTs-2.1.0-Linux/bin' >> ~/.bashrc && \
-rm ~/Downloads/ANTs-2.1.0-rc3-Linux.tar.gz && \
+yes | rm /ANTs-2.1.0-rc3-Linux.tar.gz && \
+cd / && \
 echo "## ANTs Install Complete ##" && \
 # Install C3D tool (reduced version: only c3d and c3d_affine_tool)
 echo "## Install C3D ##" && \
@@ -46,11 +46,10 @@ tar xzvf c3d-nightly-Linux-gcc64.tar.gz && \
 dirname=$(find . -maxdepth 1 -type d -name "c3d*") && \
 mv $dirname /opt/ && \
 c3dpath=$(find /opt -maxdepth 1 -type d -name "c3d*") && \
-echo 'export PATH=${PATH}:'"${c3dpath}/bin" >> ~/.bashrc && \
-rm ${c3dpath}/bin/c2d ${c3dpath}/bin/c4d ${c3dpath}/bin/c3d_gui && \
-rm -r ${c3dpath}/lib && \
-rm c3d-nightly-Linux-gcc64.tar.gz && \
-popd && \
+echo 'export PATH=${PATH}:${c3dpath}/bin' >> ~/.bashrc && \
+yes | rm ${c3dpath}/bin/c2d ${c3dpath}/bin/c4d ${c3dpath}/bin/c3d_gui && \
+yes | rm -r ${c3dpath}/lib && \
+yes | rm c3d-nightly-Linux-gcc64.tar.gz && \
 echo "## C3D Install Complete ##" && \
 # VNC Install
 echo "## VNC Install ##" && \
@@ -59,8 +58,8 @@ yum -y install xclock && \
 yum -y install xdotool && \
 echo "## VNC Install Complete ##" && \
 # Cleanup 
-rm -r ~/Downloads && \
 yum clean all && \
+yes | conda clean --all && \
 echo "## Cleanup complete ##" && \
 # setting up vnc xstartup
 mkdir ~/.vnc && \
@@ -69,4 +68,4 @@ echo 'unset DBUS_SESSION_BUS_ADDRESS' >> ~/.vnc/xstartup && \
 echo 'xsetroot -solid grey -cursor_name left_ptr' >> ~/.vnc/xstartup && \
 echo 'xeyes' >> ~/.vnc/xstartup
 
-CMD /bin/bash
+CMD /bin/bash
diff --git a/docker/Dockerfile.rttoolslite b/docker/Dockerfile.rttoolslite
@@ -1,22 +1,27 @@
-FROM centos:7.5.1804
+FROM brainiak/rtconda:1.4 
 
 RUN \
-# Yum install packages
-yum install -y git wget ca-certificates epel-release && \
+# Packages needed for FSL install
+yum install -y file which zip gzip bzip2 libpng12 libmng mesa-libGL-devel epel-release && \
+# Activate conda and bashrc 
+source ~/.bashrc && \
+conda activate rtcloud && \
+yum -y install openblas-devel.x86_64 && \
 # VNC Install
 echo "## VNC Install ##" && \
 yum -y install tigervnc-server && \
 yum -y install xclock && \
 yum -y install xdotool && \
 echo "## VNC Install Complete ##" && \
+# Cleanup 
+yum clean all && \
+yes | conda clean --all && \
+echo "## Cleanup complete ##" && \
 # setting up vnc xstartup
 mkdir ~/.vnc && \
 echo 'unset SESSION_MANAGER' >> ~/.vnc/xstartup && \ 
 echo 'unset DBUS_SESSION_BUS_ADDRESS' >> ~/.vnc/xstartup && \
 echo 'xsetroot -solid grey -cursor_name left_ptr' >> ~/.vnc/xstartup && \
-echo 'xeyes' >> ~/.vnc/xstartup && \
-# Cleanup 
-yum clean all && \
-echo "## Cleanup complete ##"
+echo 'xeyes' >> ~/.vnc/xstartup
 
-CMD /bin/bash
+CMD /bin/bash