Running DeepLabCut Training on a Linux AWS EC2

DLC_training_AWS_EC2

#!/bin/bash

########################################################################################################
##### originally forked by : https://gist.github.com/Mahedi-61/2a2f1579d4271717d421065168ce6a73 ########
##### modified by https://gist.github.com/mcvarer/30041141c8fe70ea5fe13f839330bc5a              ########
########################################################################################################

## This gist contains instructions for getting DeepLabCut set up on AWS EC2 running Ubuntu 18.04 
## Specifically: DeepLabCut v 2.2.0.6 on AWS EC2 (Ubuntu 18.04) search AWS marketplace for "ami-04cd519d2f9578053"
## The above AMI is the cheapest available to run DLC with acceptable training times. Minimum 100 GB storage req.
## An NVIDIA developer account (free) is required to download the cuDNN 8.1 library.
## It is assumed that labeling has already been performed on a local machine
## Credit to multiple sources cited above as well as of course the Mathis labs and the DLC community

### steps ####
# install and change default python to 3.8
# verify cuda-capable gpu
# download and install the nvidia cuda toolkit (v11.2) and cudnn (8.1)
# setup environmental variables
# verify the cuda installation
###

### Install python3.8.13
sudo apt install python3.8

### Check default python
python -V
python3 -V

## Update alternatives based on available specific major and minor version numbers
## The default python is python2.x.y. Available python3 is python3.6.9
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.8 13
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.6 9

## Check default python, if python 3.8 entry has * on it then hit enter
sudo update-alternatives --config python

### AMI comes with NVIDIA driver installed (450.119.03) and CUDA 11.0
### This has to be first uninstalled
sudo apt-get purge nvidia*
sudo apt remove nvidia-*
sudo rm /etc/apt/sources.list.d/cuda*
sudo apt-get autoremove && sudo apt-get autoclean
sudo rm -rf /usr/local/cuda*

### to verify your gpu is cuda enable check
lspci | grep -i nvidia

### gcc compiler is required for development using the cuda toolkit. to verify the version of gcc install enter
gcc --version

# system update
sudo apt-get update
sudo apt-get upgrade

# install other import packages
sudo apt-get install g++ freeglut3-dev build-essential libx11-dev libxmu-dev libxi-dev libglu1-mesa libglu1-mesa-dev

# first get the PPA repository driver
sudo add-apt-repository ppa:graphics-drivers/ppa
sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub
echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list
sudo apt-get update

 # installing CUDA-11.2
sudo apt-get -o Dpkg::Options::="--force-overwrite" install cuda-11-2 cuda-drivers

# setup your paths
echo 'export PATH=/usr/local/cuda-11.2/bin:$PATH' >> ~/.bashrc
echo 'export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:$LD_LIBRARY_PATH' >> ~/.bashrc
source ~/.bashrc
sudo ldconfig

# install cuDNN v8.1; On local machine, first create NVIDIA developer account (free), then proceed to cuDNN download archive
# "Download cuDNN v8.1.1 (Feburary 26th, 2021), for CUDA 11.0,11.1 and 11.2" --> "cuDNN Library for Linux (x86_64)"
# Download file and transfer to EC2 via an SFTP client like WinSCP. Place file in /home/ubuntu/
tar -xzvf cudnn-11.2-linux-ppc64le-v8.1.1.33.tgz

# copy the following files into the cuda toolkit directory.
sudo cp -P cuda/include/cudnn*.h /usr/local/cuda-11.2/include
sudo cp -P cuda/lib64/libcudnn* /usr/local/cuda-11.2/lib64/
sudo chmod a+r /usr/local/cuda-11.2/lib64/libcudnn*

# reboot to ensure recognition of driver install
sudo shutdown -r 0

# Verify the installation. NVIDIA drivers should be up-to-date and CUDA will be 11.2
nvidia-smi
nvcc -V

#Install ffmpeg
sudo apt install ffmpeg

# Create a new env and activate. Pip install DeepLabCut
conda create -n DLC python=3.8
conda activate DLC
pip install deeplabcut

# Gitclone DeepLabCut folder and cd into examples to run testscript.py; 
# "ALL DONE!!! - default cases are functional." should show up to confirm everything is working well
git clone https://github.com/DeepLabCut/DeepLabCut.git
cd DeepLabCut/examples/
python testscript.py
conda deactivate

# Use WinSCP to transfer the project folder to the EC2 and modify config.yaml file to match location on EC2

# use tmux to start a new session for your terminal so you can check the training progress at any time.
tmux new -s mysession

# cd and then ipython and proceed to use DeepLabCut as usual. If labeling was done on windows, make sure to
# deeplabcut.check_labels(path_config_file) and deeplabcut.create_training_dataset(path_config_file)
# once training begins, leave anytime using tmux
tmux detach

#to check back on training progress
tmux attach -t mysession