(wip) rework containers for 2.0

7 files changed, 39 insertions, 241 deletions

diff --git a/docker-compose.yml b/docker-compose.yml
index 2c543f9..d52637e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,11 +10,6 @@ services:
             # as long as all the Vainglory repo are checked out on the same level this will work just fine 
             # volume to map Vainsocial.com code. used by Workspace repo.
             - ../vainsocial/:/var/www/vainsocial
-            - ../apigrabber:/apps/api
-            - ../preloader:/apps/preloader
-            - ../compiler:/apps/stats
-            - ../analyzer:/apps/climate
-            - ../cruncher:/apps/cruncher
 
 ### Workspace Utilities Container ###########################
 
@@ -44,8 +39,8 @@ services:
         ports:
            - "2222:22"
         links:
-            - vaindock_postgres_raw
-            - vaindock_postgres_web
+            - vaindock_rabbitmq
+            - vaindock_postgres
         tty: true
 
 ### PHP-FPM Container #######################################
@@ -71,8 +66,7 @@ services:
             - "9000"
         links:
             - workspace
-            - vaindock_postgres_raw
-            - vaindock_postgres_web
+            - vaindock_postgres
         extra_hosts:
             # IMPORTANT: Replace with your Docker Host IP (will be appended to /etc/hosts)
             - "dockerhost:10.0.75.1"
@@ -80,11 +74,7 @@ services:
             # IMPORTANT: Set the Remote Interpreter entry matching name to `laravel`
             - PHP_IDE_CONFIG=serverName=laravel
 
-### Nginx Server Container ##################################
-### The webroot is mapped to vainsocial.dev (entry in hosts file)
-
-    nginx:
-        container_name: vaindock_nginx
+    vaindock_nginx:
         build:
             context: ./nginx
             args:
@@ -97,138 +87,39 @@ services:
         ports:
             - "88:80"
             - "443:443"
-        links:
-            - php-fpm
 
-### Python Container for apigrabber
-    apigrabber:
-        container_name: vaindock_apigrabber
-        build: ./python
-        volumes_from:
-            - applications
-        links:
-          - vaindock_postgres_raw
-        environment:
-          - POSTGRESQL_HOST=vaindock_postgres_raw
-          - POSTGRESQL_PORT=5432
-          - POSTGRESQL_DB=vainsocial-raw
-          - POSTGRESQL_USER=vainraw
-          - POSTGRESQL_PASSWORD=vainraw
-          - VAINSOCIAL_REGIONS=sg
-          - VAINSOCIAL_STARTDATE=2017-02-18T00:00:01Z
-          - VAINSOCIAL_ENDDATE=2017-02-25T00:00:00Z
-        env_file:
-            - local.env
-
-### Python Container for preloader
-    preloader:
-        container_name: vaindock_preloader
-        build: ./python
+    vaindock_apigrabber:
+        build: ../apigrabber
         volumes:
-            - ../preloader:/apps/api
-        links:
-          - vaindock_postgres_raw
+          - ../apigrabber:/code
+          - ../joblib:/code/joblib
+        depends_on:
+          - vaindock_rabbitmq
         environment:
-          - POSTGRESQL_HOST=vaindock_postgres_raw
-          - POSTGRESQL_PORT=5432
-          - POSTGRESQL_DB=vainsocial-raw
-          - POSTGRESQL_USER=vainraw
-          - POSTGRESQL_PASSWORD=vainraw
-          - VAINSOCIAL_SPIDERTHROTTLE=5
+          - RABBITMQ_HOST=vaindock_rabbitmq
         env_file:
-            - local.env
-
-### Python Container for processor
-    processor:
-        build: ./python
-        volumes:
-            - ../processor:/apps/api
-        links:
-          - vaindock_postgres_raw
-          - vaindock_postgres_web
-        environment:
-          - POSTGRESQL_SOURCE_HOST=vaindock_postgres_raw
-          - POSTGRESQL_SOURCE_PORT=5432
-          - POSTGRESQL_SOURCE_DB=vainsocial-raw
-          - POSTGRESQL_SOURCE_USER=vainraw
-          - POSTGRESQL_SOURCE_PASSWORD=vainraw
-          - POSTGRESQL_DEST_HOST=vaindock_postgres_web
-          - POSTGRESQL_DEST_PORT=5432
-          - POSTGRESQL_DEST_DB=vainsocial-web
-          - POSTGRESQL_DEST_USER=vainweb
-          - POSTGRESQL_DEST_PASSWORD=vainweb
-          - VAINSOCIAL_SPIDER=true
-          - VAINSOCIAL_ANALYZE=true
-
-
-### Python Container for compiler
-    compiler:
-        build: ./python
-        volumes:
-            - ../compiler:/apps/api
-        links:
-          - vaindock_postgres_raw
-          - vaindock_postgres_web
-        environment:
-          - POSTGRESQL_SOURCE_HOST=vaindock_postgres_raw
-          - POSTGRESQL_SOURCE_PORT=5432
-          - POSTGRESQL_SOURCE_DB=vainsocial-raw
-          - POSTGRESQL_SOURCE_USER=vainraw
-          - POSTGRESQL_SOURCE_PASSWORD=vainraw
-          - POSTGRESQL_DEST_HOST=vaindock_postgres_web
-          - POSTGRESQL_DEST_PORT=5432
-          - POSTGRESQL_DEST_DB=vainsocial-web
-          - POSTGRESQL_DEST_USER=vainweb
-          - POSTGRESQL_DEST_PASSWORD=vainweb
+          - local.env
 
-### Python Container for analyzer
-    analyzer:
-        build: ./python
+    vaindock_processor:
+        build: ../processor
         volumes:
-            - ../analyzer:/apps/api
-        links:
-          - vaindock_postgres_raw
-          - vaindock_postgres_web
+          - ../processor:/code
+          - ../joblib:/code/joblib
+        depends_on:
+          - vaindock_rabbitmq
+          - vaindock_postgres
         environment:
-          - POSTGRESQL_SOURCE_HOST=vaindock_postgres_raw
-          - POSTGRESQL_SOURCE_PORT=5432
-          - POSTGRESQL_SOURCE_DB=vainsocial-raw
-          - POSTGRESQL_SOURCE_USER=vainraw
-          - POSTGRESQL_SOURCE_PASSWORD=vainraw
-          - POSTGRESQL_DEST_HOST=vaindock_postgres_web
-          - POSTGRESQL_DEST_PORT=5432
-          - POSTGRESQL_DEST_DB=vainsocial-web
-          - POSTGRESQL_DEST_USER=vainweb
-          - POSTGRESQL_DEST_PASSWORD=vainweb
+          - RABBITMQ_HOST=vaindock_rabbitmq
+          - POSTGRESQL_HOST=vaindock_postgres
+          - POSTGRESQL_DB=vainsocial-web
+          - POSTGRESQL_USER=vainweb
+          - POSTGRESQL_PASSWORD=vainweb
 
-### Python Container for cruncher
-    cruncher:
-        build: ./python
-        volumes:
-            - ../cruncher:/apps/api
-        links:
-          - vaindock_postgres_raw
-          - vaindock_postgres_web
-        environment:
-          - POSTGRESQL_SOURCE_HOST=vaindock_postgres_raw
-          - POSTGRESQL_SOURCE_PORT=5432
-          - POSTGRESQL_SOURCE_DB=vainsocial-raw
-          - POSTGRESQL_SOURCE_USER=vainraw
-          - POSTGRESQL_SOURCE_PASSWORD=vainraw
-          - POSTGRESQL_DEST_HOST=vaindock_postgres_web
-          - POSTGRESQL_DEST_PORT=5432
-          - POSTGRESQL_DEST_DB=vainsocial-web
-          - POSTGRESQL_DEST_USER=vainweb
-          - POSTGRESQL_DEST_PASSWORD=vainweb
 
-
-### PostgreSQL Container - Web ####################################
-
-    vaindock_postgres_web:
-        container_name: vaindock_postgres_web
-        build: ./postgres
+    vaindock_postgres:
+        image: postgres:9.6-alpine
         volumes:
-            - vaindock_pg_web:/var/lib/postgresql/data
+            - vaindock_postgres-data:/var/lib/postgresql/data
         ports:
             - "5432:5432"
         environment:
@@ -236,57 +127,24 @@ services:
             POSTGRES_USER: vainweb
             POSTGRES_PASSWORD: vainweb
 
-### PostgreSQL Container - Raw ####################################
 
-    vaindock_postgres_raw:
-        container_name: vaindock_postgres_raw
-        build: ./postgres
+    vaindock_rabbitmq:
+        image: rabbitmq:3-management-alpine
         volumes:
-            - vaindock_pg_raw:/var/lib/postgresql/data
+          - vaindock_rabbitmq-data:/var/lib/rabbitmq
         ports:
-            - "5433:5432"
+          - "4369:4369"
+          - "5671:5671"
+          - "5672:5672"
+          - "25672:25672"
+          - "8881:15672"  # management guest/guest
         environment:
-            POSTGRES_DB: vainsocial-raw
-            POSTGRES_USER: vainraw
-            POSTGRES_PASSWORD: vainraw
-
-### pgAdmin Container #######################################
-
-    pgadmin:
-        container_name: vaindock_pgadmin
-        build: ./pgadmin
-        ports:
-            - "5050:5050"
-        links:
-            - vaindock_postgres_raw
-            - vaindock_postgres_web
-
-
-### Volumes Setup ###########################################
+          - RABBITMQ_HIPE_COMPILE=1
 
 volumes:
-    vaindock_pg_raw:
+    vaindock_postgres-data:
         driver: "local"
-    vaindock_pg_web:
+    vaindock_rabbitmq-data:
         driver: "local"
     sessions:    ## nothing is connected to this (- ./data/sessions:/sessions)
         driver: "local"
-
-
-
-### Apache Server Container #################################
-
-    # apache2:
-    #     build:
-    #         context: ./apache2
-    #         args:
-    #             - PHP_SOCKET=php-fpm:9000
-    #     volumes_from:
-    #         - applications
-    #     volumes:
-    #         - ./logs/apache2:/var/log/apache2
-    #     ports:
-    #         - "80:80"
-    #         - "443:443"
-    #     links:
-    #         - php-fpm
diff --git a/local.env b/local.env
index d84ae1f..211b76d 100644
--- a/local.env
+++ b/local.env
@@ -1 +1 @@
-VAINSOCIAL_APITOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJiZjQ1MWE5MC1kZDgyLTAxMzQtY2E0YS0wMjQyYWMxMTAwMDYiLCJpc3MiOiJnYW1lbG9ja2VyIiwib3JnIjoic2NobmVlZnV4LXZhaW5nbG9yeWFwaV92YWluc29jaWFsLWRldi1zY2huZWVmdXgteHl6IiwiYXBwIjoiYmY0MzZiMjAtZGQ4Mi0wMTM0LWNhNDktMDI0MmFjMTEwMDA2IiwicHViIjoic2VtYyIsInRpdGxlIjoidmFpbmdsb3J5Iiwic2NvcGUiOiJjb21tdW5pdHkiLCJsaW1pdCI6MTB9.7ubbpc1INytkK0Z7kZ2fwz2OJBRagJhIhwVJz6mUUow
+MADGLORY_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJiZjQ1MWE5MC1kZDgyLTAxMzQtY2E0YS0wMjQyYWMxMTAwMDYiLCJpc3MiOiJnYW1lbG9ja2VyIiwib3JnIjoic2NobmVlZnV4LXZhaW5nbG9yeWFwaV92YWluc29jaWFsLWRldi1zY2huZWVmdXgteHl6IiwiYXBwIjoiYmY0MzZiMjAtZGQ4Mi0wMTM0LWNhNDktMDI0MmFjMTEwMDA2IiwicHViIjoic2VtYyIsInRpdGxlIjoidmFpbmdsb3J5Iiwic2NvcGUiOiJjb21tdW5pdHkiLCJsaW1pdCI6MTB9.7ubbpc1INytkK0Z7kZ2fwz2OJBRagJhIhwVJz6mUUow
diff --git a/pgadmin/Dockerfile b/pgadmin/Dockerfile
deleted file mode 100644
index 2d34c22..0000000
--- a/pgadmin/Dockerfile
+++ /dev/null
@@ -1,5 +0,0 @@
-FROM fenglc/pgadmin4
-
-MAINTAINER Bo-Yi Wu <appleboy.tw@gmail.com>
-
-EXPOSE 5050
diff --git a/postgres/Dockerfile b/postgres/Dockerfile
deleted file mode 100644
index 249cee5..0000000
--- a/postgres/Dockerfile
+++ /dev/null
@@ -1,8 +0,0 @@
-FROM postgres:latest
-
-MAINTAINER Ben M <git@bmagg.com>
-
-CMD ["postgres"]
-
-EXPOSE 5432
-EXPOSE 5433
\ No newline at end of file
diff --git a/python/Dockerfile b/python/Dockerfile
deleted file mode 100644
index ef16764..0000000
--- a/python/Dockerfile
+++ /dev/null
@@ -1,23 +0,0 @@
-FROM python:3.6
-ADD requirements.txt /
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        build-essential \
-        curl \
-        libfreetype6-dev \
-        libpng12-dev \
-        libzmq3-dev \
-        pkg-config \
-        rsync \
-        software-properties-common \
-        unzip \
-        && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN pip --no-cache-dir install numpy
-
-RUN pip install -r /requirements.txt
-#RUN pip install tensorflow==1.0.1
-RUN pip install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.0.1-cp36-cp36m-linux_x86_64.whl
-CMD ["python3", "/apps/api/api.py"]
diff --git a/python/requirements.txt b/python/requirements.txt
deleted file mode 100644
index 7eb6f20..0000000
--- a/python/requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-aiohttp==1.2.0
-appdirs==1.4.0
-async-timeout==1.1.0
-chardet==2.3.0
-multidict==2.1.4
-packaging==16.8
-pyparsing==2.1.10
-six==1.10.0
-yarl==0.9.0
-asyncpg==0.9.0
-numpy==1.12.0
-protobuf==3.2.0
-psycopg2==2.7
diff --git a/redis/Dockerfile b/redis/Dockerfile
deleted file mode 100644
index e8fedab..0000000
--- a/redis/Dockerfile
+++ /dev/null
@@ -1,11 +0,0 @@
-FROM redis:latest
-
-MAINTAINER Mahmoud Zalt <mahmoud@zalt.me>
-
-#COPY redis.conf /usr/local/etc/redis/redis.conf
-
-VOLUME /data
-
-EXPOSE 6379
-
-CMD ["redis-server"]