Docker integration (#4)

* fix(Docker Integration):

Cleaned unused files
Updated Trash Collector
Updated Docker Compose
Updated Start Script

* feature(microservice rpc docker):

* feature(microservice rpc docker):

* fix(Added API Environment):

Added environment variable for docker compose

safeshare/docker-compose.yml

@@ -1,54 +1,35 @@
 version: '3'
 
 services:
-  db:
-    image: mariadb:10.5
-    command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW
-    restart: always
-    volumes:
-      - db_data:/var/lib/mysql
-    networks:
-      - dbnet
-    environment:
-      - MYSQL_DATABASE=safeShare
-      - MYSQL_USER=user
-      - MYSQL_ROOT_PASSWORD=password
-      - MYSQL_PASSWORD=password
-    expose:
-      - 3306
-
   redis:
     image: redis:latest
-    command: redis-server --requirepass password
+    command: redis-server
     restart: always
     networks:
       - dbnet
     expose:
       - 6379
     volumes:
-        - redis_data:/data
+      - redis_data:/data
 
   safeshare:
     build:
       context: .
       dockerfile: Dockerfile
     restart:
-        always
+      always
     ports:
       - "8000:8000"  # Map container port 8000 to host port 8000
     environment:
       - DEBUG=True
       - SECRET_KEY=A_RANDOM_SECRET_KEY
-      - DB_NAME=testSite
-      - DB_USER=user
-      - DB_PASSWORD=password
-      - DB_HOST=db
-      - DB_PORT=3306
       - ALLOWED_HOSTS=*
       - CACHE=True
-      - REDIS_URL=redis://:password@redis:6379/0
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_DB=0
+      - TRASH_TIMEOUT=60
     depends_on:
-      - db
       - redis
     networks:
       - dbnet
@@ -58,16 +39,28 @@ services:
       context: ./safeshare-frontend
       dockerfile: Dockerfile
     restart:
-        always
+      always
     ports:
       - "3000:3000"  # Map container port 3000 to host port 3000
     environment:
       - REACT_APP_API_URL=http://localhost:8000/api
     networks:
       - dbnet
-volumes:
-    db_data:
-    redis_data:
 
+  safeshare-virus:
+    build:
+      context: ./safeshare_vdb
+      dockerfile: Dockerfile
+    restart:
+      always
+    ports:
+      - "50051:50051"
+    environment:
+      - API_TOKEN=YOUR_API_TOKEN
+    networks:
+      - dbnet
+
+volumes:
+  redis_data:
 networks:
   dbnet:

safeshare/safeshare/settings.py

@@ -39,6 +39,7 @@ ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*'])
 REDIS_HOST = env.str('REDIS_HOST', default='localhost')
 REDIS_PORT = env.str('REDIS_PORT', default='6379')
 REDIS_DB = env.str('REDIS_DB', default='0')
+TRASH_TIMEOUT = env.int('TRASH_TIMEOUT', default=60)
 
 # Application definition
 
@@ -136,15 +137,13 @@ if os.getenv('GITHUB_WORKFLOW'):
 else:
     DATABASES = {
         'default': {
-            'ENGINE': 'django.db.backends.mysql',
+            'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': env.str('DB_NAME', 'safeshare'),
+            # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
-            'USER': env.str('DB_USER', 'mariadb'),
+            'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),  # Or path to database file if using sqlite3.
-            'PASSWORD': env.str('DB_PASSWORD', 'mariadb'),
+            'USER': '',  # Not used with sqlite3.
-            'HOST': env.str('DB_HOST', 'localhost'),
+            'PASSWORD': '',  # Not used with sqlite3.
-            'PORT': env.str('DB_PORT', '3306'),
+            'HOST': '',  # Set to empty string for localhost. Not used with sqlite3.
-            'OPTIONS': {
+            'PORT': '',  # Set to empty string for default. Not used with sqlite3.
-                'init_command': "SET sql_mode='STRICT_TRANS_TABLES', innodb_strict_mode=1",
-            },
         }
     }
 

safeshare/safeshare/urls.py

@@ -17,9 +17,8 @@ Including another URLconf
 from django.contrib import admin
 from django.urls import path, include
 from safeshare_app.utils.TrashCollector import TrashCollector
+import threading
 
-trash_collector = TrashCollector()
-trash_collector.start()
 
 urlpatterns = [
     path('admin/', admin.site.urls),

safeshare/safeshare_app/management/commands/start_trash_collector.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+from safeshare_app.utils.TrashCollector.TrashCollector import TrashCollector
+
+class Command(BaseCommand):
+    help = 'Start the trash collector'
+
+    def handle(self, *args, **options):
+        trash_collector = TrashCollector()
+        trash_collector.start()

safeshare/safeshare_app/utils/TrashCollector/TrashCollector.py

@@ -1,7 +1,8 @@
-import threading
 import os
-import redis
+import threading
+
 import environ
+import redis
 from django.conf import settings
 
 
@@ -11,13 +12,11 @@ class TrashCollector:
         self.thread = threading.Thread(target=self.run)
         self.media_root = settings.MEDIA_ROOT
 
-        environ.Env.read_env(os.path.join('safeshare', 'env'))
-        self.env = environ.Env()
         # Connect to Redis
         self.redis = redis.StrictRedis(
-            host=self.env.str('REDIS_HOST', default='localhost'),
+            host=settings.REDIS_HOST,
-            port=self.env.str('REDIS_PORT', default=6379),
+            port=settings.REDIS_PORT,
-            db=self.env.str('REDIS_DB', default=0),
+            db=settings.REDIS_DB,
         )
 
     def start(self):
@@ -67,4 +66,15 @@ class TrashCollector:
                                 print(e)
 
             # Sleep for a specific interval in seconds
-            self.stop_event.wait(timeout=self.env.int('TRASH_TIMEOUT', default=60))
+            self.stop_event.wait(timeout=settings.TRASH_TIMEOUT)
+
+
+if __name__ == '__main__':
+    trash_collector = TrashCollector()
+    try:
+        print("Starting trash collector")
+        trash_collector.start()
+        print("Trash collector started")
+    except KeyboardInterrupt:
+        trash_collector.stop()
+        print("Trash collector stopped")

safeshare/safeshare_app/utils/virusRPC/client.py

@@ -1,20 +0,0 @@
-import grpc
-
-import scan_pb2_grpc
-import scan_pb2
-
-
-class Client:
-    def __init__(self):
-        self.channel = grpc.insecure_channel("localhost:50051")
-        self.stub = scan_pb2_grpc.VirusScanServiceStub(self.channel)
-
-    def ScanFile(self, sha_256_id: str):
-        response = self.stub.ScanFile(scan_pb2.ScanFileRequest(file_SHA256=sha_256_id))
-        print(response)
-
-
-if __name__ == "__main__":
-    client = Client()
-    id = "15e4313dddb45875ed67d1ab25f1f5b76f0b3a23e4fa9308c521e3fb30068028"
-    client.ScanFile(id)

safeshare/safeshare_app/utils/virusRPC/scan.proto

@@ -1,24 +0,0 @@
-syntax = "proto3";
-
-package virusscan;
-
-service VirusScanService {
-  rpc ScanFile (ScanFileRequest) returns (ScanFileResponse) {}
-}
-
-message ScanFileRequest {
-  string file_name = 1;
-  string file_SHA256 = 2;
-  string file_SHA1 = 3;
-  string file_MD5 = 4;
-}
-
-message ScanFileResponse {
-  string file_name = 1;
-  string file_SHA256 = 2;
-  string file_SHA1 = 3;
-  string file_MD5 = 4;
-  bool is_infected = 5;
-  string scan_result = 6;
-  string scan_result_detail = 7;
-}

safeshare/safeshare_app/utils/virusRPC/scan_pb2.py

@@ -1,29 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: scan.proto
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nscan.proto\x12\tvirusscan\"^\n\x0fScanFileRequest\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ile_SHA256\x18\x02 \x01(\t\x12\x11\n\tfile_SHA1\x18\x03 \x01(\t\x12\x10\n\x08\x66ile_MD5\x18\x04 \x01(\t\"\xa5\x01\n\x10ScanFileResponse\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ile_SHA256\x18\x02 \x01(\t\x12\x11\n\tfile_SHA1\x18\x03 \x01(\t\x12\x10\n\x08\x66ile_MD5\x18\x04 \x01(\t\x12\x13\n\x0bis_infected\x18\x05 \x01(\x08\x12\x13\n\x0bscan_result\x18\x06 \x01(\t\x12\x1a\n\x12scan_result_detail\x18\x07 \x01(\t2Y\n\x10VirusScanService\x12\x45\n\x08ScanFile\x12\x1a.virusscan.ScanFileRequest\x1a\x1b.virusscan.ScanFileResponse\"\x00\x62\x06proto3')
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scan_pb2', _globals)
-if _descriptor._USE_C_DESCRIPTORS == False:
-  DESCRIPTOR._options = None
-  _globals['_SCANFILEREQUEST']._serialized_start=25
-  _globals['_SCANFILEREQUEST']._serialized_end=119
-  _globals['_SCANFILERESPONSE']._serialized_start=122
-  _globals['_SCANFILERESPONSE']._serialized_end=287
-  _globals['_VIRUSSCANSERVICE']._serialized_start=289
-  _globals['_VIRUSSCANSERVICE']._serialized_end=378
-# @@protoc_insertion_point(module_scope)

safeshare/safeshare_app/utils/virusRPC/scan_pb2.pyi

@@ -1,35 +0,0 @@
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import message as _message
-from typing import ClassVar as _ClassVar, Optional as _Optional
-
-DESCRIPTOR: _descriptor.FileDescriptor
-
-class ScanFileRequest(_message.Message):
-    __slots__ = ["file_name", "file_SHA256", "file_SHA1", "file_MD5"]
-    FILE_NAME_FIELD_NUMBER: _ClassVar[int]
-    FILE_SHA256_FIELD_NUMBER: _ClassVar[int]
-    FILE_SHA1_FIELD_NUMBER: _ClassVar[int]
-    FILE_MD5_FIELD_NUMBER: _ClassVar[int]
-    file_name: str
-    file_SHA256: str
-    file_SHA1: str
-    file_MD5: str
-    def __init__(self, file_name: _Optional[str] = ..., file_SHA256: _Optional[str] = ..., file_SHA1: _Optional[str] = ..., file_MD5: _Optional[str] = ...) -> None: ...
-
-class ScanFileResponse(_message.Message):
-    __slots__ = ["file_name", "file_SHA256", "file_SHA1", "file_MD5", "is_infected", "scan_result", "scan_result_detail"]
-    FILE_NAME_FIELD_NUMBER: _ClassVar[int]
-    FILE_SHA256_FIELD_NUMBER: _ClassVar[int]
-    FILE_SHA1_FIELD_NUMBER: _ClassVar[int]
-    FILE_MD5_FIELD_NUMBER: _ClassVar[int]
-    IS_INFECTED_FIELD_NUMBER: _ClassVar[int]
-    SCAN_RESULT_FIELD_NUMBER: _ClassVar[int]
-    SCAN_RESULT_DETAIL_FIELD_NUMBER: _ClassVar[int]
-    file_name: str
-    file_SHA256: str
-    file_SHA1: str
-    file_MD5: str
-    is_infected: bool
-    scan_result: str
-    scan_result_detail: str
-    def __init__(self, file_name: _Optional[str] = ..., file_SHA256: _Optional[str] = ..., file_SHA1: _Optional[str] = ..., file_MD5: _Optional[str] = ..., is_infected: bool = ..., scan_result: _Optional[str] = ..., scan_result_detail: _Optional[str] = ...) -> None: ...

safeshare/safeshare_app/utils/virusRPC/scan_pb2_grpc.py

@@ -1,66 +0,0 @@
-# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
-"""Client and server classes corresponding to protobuf-defined services."""
-import grpc
-
-import scan_pb2 as scan__pb2
-
-
-class VirusScanServiceStub(object):
-    """Missing associated documentation comment in .proto file."""
-
-    def __init__(self, channel):
-        """Constructor.
-
-        Args:
-            channel: A grpc.Channel.
-        """
-        self.ScanFile = channel.unary_unary(
-                '/virusscan.VirusScanService/ScanFile',
-                request_serializer=scan__pb2.ScanFileRequest.SerializeToString,
-                response_deserializer=scan__pb2.ScanFileResponse.FromString,
-                )
-
-
-class VirusScanServiceServicer(object):
-    """Missing associated documentation comment in .proto file."""
-
-    def ScanFile(self, request, context):
-        """Missing associated documentation comment in .proto file."""
-        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-        context.set_details('Method not implemented!')
-        raise NotImplementedError('Method not implemented!')
-
-
-def add_VirusScanServiceServicer_to_server(servicer, server):
-    rpc_method_handlers = {
-            'ScanFile': grpc.unary_unary_rpc_method_handler(
-                    servicer.ScanFile,
-                    request_deserializer=scan__pb2.ScanFileRequest.FromString,
-                    response_serializer=scan__pb2.ScanFileResponse.SerializeToString,
-            ),
-    }
-    generic_handler = grpc.method_handlers_generic_handler(
-            'virusscan.VirusScanService', rpc_method_handlers)
-    server.add_generic_rpc_handlers((generic_handler,))
-
-
- # This class is part of an EXPERIMENTAL API.
-class VirusScanService(object):
-    """Missing associated documentation comment in .proto file."""
-
-    @staticmethod
-    def ScanFile(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/virusscan.VirusScanService/ScanFile',
-            scan__pb2.ScanFileRequest.SerializeToString,
-            scan__pb2.ScanFileResponse.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

safeshare/safeshare_app/utils/virusRPC/server.py

@@ -1,61 +0,0 @@
-import grpc
-import scan_pb2
-import scan_pb2_grpc
-from concurrent import futures
-import requests
-
-apiKey = "" # API key from VirusTotal
-
-headers = {
-    "accept": "application/json",
-    "x-apikey": apiKey
-}
-
-
-class VirusScanServicer(scan_pb2_grpc.VirusScanServiceServicer):
-    url = "https://www.virustotal.com/api/v3/files/"
-
-    def ScanFile(self, request, context):
-        result = self.ScanSHA(request.file_SHA256)
-        return result
-
-    def ScanSHA(self, sha256_hash: str) -> scan_pb2.ScanFileResponse:
-        self.url += sha256_hash
-
-        response = requests.get(self.url, headers=headers)
-        data = response.json()["data"]
-        result = scan_pb2.ScanFileResponse()
-
-        result.is_infected = data["attributes"]["last_analysis_stats"]["malicious"] > 0
-
-        result.file_name = data["attributes"]["names"][0]
-        result.file_SHA1 = data["attributes"]["sha1"]
-        result.file_SHA256 = data["attributes"]["sha256"]
-
-        return result
-
-
-class VirusScanServer:
-    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
-    scan_pb2_grpc.add_VirusScanServiceServicer_to_server(VirusScanServicer(), server)
-    server.add_insecure_port("[::]:50051")
-
-    def serve(self):
-        try:
-            self.server.start()
-            print("Server started, listening on 50051")
-            self.server.wait_for_termination()
-        except KeyboardInterrupt:
-            print("Server stopped")
-            self.server.stop(0)
-
-    def __del__(self):
-        self.server.stop(0)
-
-    def __enter__(self):
-        return self
-
-
-if __name__ == "__main__":
-    cal = VirusScanServer()
-    cal.serve()

safeshare/safeshare_vdb/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11
+LABEL authors="junwei"
+
+# Copy the current directory contents into the container at /app
+COPY . /app
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Install any needed packages for grpc and environ
+RUN pip install grpcio grpcio-tools django-environ requests boto3
+
+# Start Server at server.py
+CMD ["python", "server.py"]

safeshare/start.sh

@@ -1,11 +1,16 @@
 #!/bin/bash
 
-# Wait until Database is ready
+# Migrate the database (if needed)
-while ! nc -z db 3306; do sleep 1; done
-
 python manage.py migrate --noinput
 
-# Seed the database with initial data
+# Start your Django server in the background
-#python manage.py loaddata study_together_app/fixtures/*
+python manage.py runserver 0.0.0.0:8000 &
 
-python manage.py runserver 0.0.0.0:8000
+# Sleep briefly to allow the Django server to start (you can adjust the sleep duration as needed)
+sleep 2
+
+# Start the custom management command to run the trash collector
+python manage.py start_trash_collector
+
+# Optionally, you can monitor the logs in real-time if needed
+tail -f django_server.log trash_collector.log