Implement MPI as tensorflow operation
cd mpi-tensorflow
# build inplace
python setup.py build_ext --inplace
# install
python setup.py install
# use system ld rather than anaconda's ld
using classic MPI_allreduce
mpirun -np 4 python example_distributed_allreduce_optimizer.py
compress gradient using HSQ in the paper is under review
mpirun -np 4 python example_distributed_hsq_optimizer.py
compress gradient using HSQ in the paper is under review
mpirun -np 4 python example_distributed_nn_optimizer.py
compress gradient using QSGD: Communication-Efficient SGD via Gradient Quantization and Encoding
mpirun -np 4 python example_distributed_qsgd_optimizer.py