From e534fa4e13a2540e0e7a0bf584fd7ccdfaadce75 Mon Sep 17 00:00:00 2001 From: Devoalda <50709414+Devoalda@users.noreply.github.com> Date: Mon, 30 Oct 2023 08:22:11 +0800 Subject: [PATCH] Docker integration (#4) * fix(Docker Integration): Cleaned unused files Updated Trash Collector Updated Docker Compose Updated Start Script * feature(microservice rpc docker): * feature(microservice rpc docker): * fix(Added API Environment): Added environment variable for docker compose --- safeshare/docker-compose.yml | 53 +++++++-------- safeshare/safeshare/settings.py | 17 +++-- safeshare/safeshare/urls.py | 3 +- .../commands}/__init__.py | 0 .../commands/start_trash_collector.py | 9 +++ .../{ => TrashCollector}/TrashCollector.py | 26 +++++--- .../safeshare_app/utils/virusRPC/client.py | 20 ------ .../safeshare_app/utils/virusRPC/scan.proto | 24 ------- .../safeshare_app/utils/virusRPC/scan_pb2.py | 29 -------- .../safeshare_app/utils/virusRPC/scan_pb2.pyi | 35 ---------- .../utils/virusRPC/scan_pb2_grpc.py | 66 ------------------- .../safeshare_app/utils/virusRPC/server.py | 61 ----------------- safeshare/safeshare_vdb/Dockerfile | 14 ++++ safeshare/start.sh | 17 +++-- 14 files changed, 84 insertions(+), 290 deletions(-) rename safeshare/safeshare_app/{utils/virusRPC => management/commands}/__init__.py (100%) create mode 100644 safeshare/safeshare_app/management/commands/start_trash_collector.py rename safeshare/safeshare_app/utils/{ => TrashCollector}/TrashCollector.py (82%) delete mode 100644 safeshare/safeshare_app/utils/virusRPC/client.py delete mode 100644 safeshare/safeshare_app/utils/virusRPC/scan.proto delete mode 100644 safeshare/safeshare_app/utils/virusRPC/scan_pb2.py delete mode 100644 safeshare/safeshare_app/utils/virusRPC/scan_pb2.pyi delete mode 100644 safeshare/safeshare_app/utils/virusRPC/scan_pb2_grpc.py delete mode 100644 safeshare/safeshare_app/utils/virusRPC/server.py create mode 100644 safeshare/safeshare_vdb/Dockerfile mode change 100644 => 100755 safeshare/start.sh diff --git a/safeshare/docker-compose.yml b/safeshare/docker-compose.yml index 6952489..66b1485 100644 --- a/safeshare/docker-compose.yml +++ b/safeshare/docker-compose.yml @@ -1,54 +1,35 @@ version: '3' services: - db: - image: mariadb:10.5 - command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW - restart: always - volumes: - - db_data:/var/lib/mysql - networks: - - dbnet - environment: - - MYSQL_DATABASE=safeShare - - MYSQL_USER=user - - MYSQL_ROOT_PASSWORD=password - - MYSQL_PASSWORD=password - expose: - - 3306 - redis: image: redis:latest - command: redis-server --requirepass password + command: redis-server restart: always networks: - dbnet expose: - 6379 volumes: - - redis_data:/data + - redis_data:/data safeshare: build: context: . dockerfile: Dockerfile restart: - always + always ports: - "8000:8000" # Map container port 8000 to host port 8000 environment: - DEBUG=True - SECRET_KEY=A_RANDOM_SECRET_KEY - - DB_NAME=testSite - - DB_USER=user - - DB_PASSWORD=password - - DB_HOST=db - - DB_PORT=3306 - ALLOWED_HOSTS=* - CACHE=True - - REDIS_URL=redis://:password@redis:6379/0 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_DB=0 + - TRASH_TIMEOUT=60 depends_on: - - db - redis networks: - dbnet @@ -58,16 +39,28 @@ services: context: ./safeshare-frontend dockerfile: Dockerfile restart: - always + always ports: - "3000:3000" # Map container port 3000 to host port 3000 environment: - REACT_APP_API_URL=http://localhost:8000/api networks: - dbnet -volumes: - db_data: - redis_data: + safeshare-virus: + build: + context: ./safeshare_vdb + dockerfile: Dockerfile + restart: + always + ports: + - "50051:50051" + environment: + - API_TOKEN=YOUR_API_TOKEN + networks: + - dbnet + +volumes: + redis_data: networks: dbnet: diff --git a/safeshare/safeshare/settings.py b/safeshare/safeshare/settings.py index 9f89d03..fcefd64 100644 --- a/safeshare/safeshare/settings.py +++ b/safeshare/safeshare/settings.py @@ -39,6 +39,7 @@ ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*']) REDIS_HOST = env.str('REDIS_HOST', default='localhost') REDIS_PORT = env.str('REDIS_PORT', default='6379') REDIS_DB = env.str('REDIS_DB', default='0') +TRASH_TIMEOUT = env.int('TRASH_TIMEOUT', default=60) # Application definition @@ -136,15 +137,13 @@ if os.getenv('GITHUB_WORKFLOW'): else: DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': env.str('DB_NAME', 'safeshare'), - 'USER': env.str('DB_USER', 'mariadb'), - 'PASSWORD': env.str('DB_PASSWORD', 'mariadb'), - 'HOST': env.str('DB_HOST', 'localhost'), - 'PORT': env.str('DB_PORT', '3306'), - 'OPTIONS': { - 'init_command': "SET sql_mode='STRICT_TRANS_TABLES', innodb_strict_mode=1", - }, + 'ENGINE': 'django.db.backends.sqlite3', + # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), # Or path to database file if using sqlite3. + 'USER': '', # Not used with sqlite3. + 'PASSWORD': '', # Not used with sqlite3. + 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string for default. Not used with sqlite3. } } diff --git a/safeshare/safeshare/urls.py b/safeshare/safeshare/urls.py index d9720e5..65138e0 100644 --- a/safeshare/safeshare/urls.py +++ b/safeshare/safeshare/urls.py @@ -17,9 +17,8 @@ Including another URLconf from django.contrib import admin from django.urls import path, include from safeshare_app.utils.TrashCollector import TrashCollector +import threading -trash_collector = TrashCollector() -trash_collector.start() urlpatterns = [ path('admin/', admin.site.urls), diff --git a/safeshare/safeshare_app/utils/virusRPC/__init__.py b/safeshare/safeshare_app/management/commands/__init__.py similarity index 100% rename from safeshare/safeshare_app/utils/virusRPC/__init__.py rename to safeshare/safeshare_app/management/commands/__init__.py diff --git a/safeshare/safeshare_app/management/commands/start_trash_collector.py b/safeshare/safeshare_app/management/commands/start_trash_collector.py new file mode 100644 index 0000000..e175dbf --- /dev/null +++ b/safeshare/safeshare_app/management/commands/start_trash_collector.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from safeshare_app.utils.TrashCollector.TrashCollector import TrashCollector + +class Command(BaseCommand): + help = 'Start the trash collector' + + def handle(self, *args, **options): + trash_collector = TrashCollector() + trash_collector.start() diff --git a/safeshare/safeshare_app/utils/TrashCollector.py b/safeshare/safeshare_app/utils/TrashCollector/TrashCollector.py similarity index 82% rename from safeshare/safeshare_app/utils/TrashCollector.py rename to safeshare/safeshare_app/utils/TrashCollector/TrashCollector.py index 28e4eff..fa8a41d 100644 --- a/safeshare/safeshare_app/utils/TrashCollector.py +++ b/safeshare/safeshare_app/utils/TrashCollector/TrashCollector.py @@ -1,7 +1,8 @@ -import threading import os -import redis +import threading + import environ +import redis from django.conf import settings @@ -11,13 +12,11 @@ class TrashCollector: self.thread = threading.Thread(target=self.run) self.media_root = settings.MEDIA_ROOT - environ.Env.read_env(os.path.join('safeshare', 'env')) - self.env = environ.Env() # Connect to Redis self.redis = redis.StrictRedis( - host=self.env.str('REDIS_HOST', default='localhost'), - port=self.env.str('REDIS_PORT', default=6379), - db=self.env.str('REDIS_DB', default=0), + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DB, ) def start(self): @@ -67,4 +66,15 @@ class TrashCollector: print(e) # Sleep for a specific interval in seconds - self.stop_event.wait(timeout=self.env.int('TRASH_TIMEOUT', default=60)) + self.stop_event.wait(timeout=settings.TRASH_TIMEOUT) + + +if __name__ == '__main__': + trash_collector = TrashCollector() + try: + print("Starting trash collector") + trash_collector.start() + print("Trash collector started") + except KeyboardInterrupt: + trash_collector.stop() + print("Trash collector stopped") diff --git a/safeshare/safeshare_app/utils/virusRPC/client.py b/safeshare/safeshare_app/utils/virusRPC/client.py deleted file mode 100644 index 98f1043..0000000 --- a/safeshare/safeshare_app/utils/virusRPC/client.py +++ /dev/null @@ -1,20 +0,0 @@ -import grpc - -import scan_pb2_grpc -import scan_pb2 - - -class Client: - def __init__(self): - self.channel = grpc.insecure_channel("localhost:50051") - self.stub = scan_pb2_grpc.VirusScanServiceStub(self.channel) - - def ScanFile(self, sha_256_id: str): - response = self.stub.ScanFile(scan_pb2.ScanFileRequest(file_SHA256=sha_256_id)) - print(response) - - -if __name__ == "__main__": - client = Client() - id = "15e4313dddb45875ed67d1ab25f1f5b76f0b3a23e4fa9308c521e3fb30068028" - client.ScanFile(id) diff --git a/safeshare/safeshare_app/utils/virusRPC/scan.proto b/safeshare/safeshare_app/utils/virusRPC/scan.proto deleted file mode 100644 index d002fd7..0000000 --- a/safeshare/safeshare_app/utils/virusRPC/scan.proto +++ /dev/null @@ -1,24 +0,0 @@ -syntax = "proto3"; - -package virusscan; - -service VirusScanService { - rpc ScanFile (ScanFileRequest) returns (ScanFileResponse) {} -} - -message ScanFileRequest { - string file_name = 1; - string file_SHA256 = 2; - string file_SHA1 = 3; - string file_MD5 = 4; -} - -message ScanFileResponse { - string file_name = 1; - string file_SHA256 = 2; - string file_SHA1 = 3; - string file_MD5 = 4; - bool is_infected = 5; - string scan_result = 6; - string scan_result_detail = 7; -} diff --git a/safeshare/safeshare_app/utils/virusRPC/scan_pb2.py b/safeshare/safeshare_app/utils/virusRPC/scan_pb2.py deleted file mode 100644 index d42285f..0000000 --- a/safeshare/safeshare_app/utils/virusRPC/scan_pb2.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: scan.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nscan.proto\x12\tvirusscan\"^\n\x0fScanFileRequest\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ile_SHA256\x18\x02 \x01(\t\x12\x11\n\tfile_SHA1\x18\x03 \x01(\t\x12\x10\n\x08\x66ile_MD5\x18\x04 \x01(\t\"\xa5\x01\n\x10ScanFileResponse\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ile_SHA256\x18\x02 \x01(\t\x12\x11\n\tfile_SHA1\x18\x03 \x01(\t\x12\x10\n\x08\x66ile_MD5\x18\x04 \x01(\t\x12\x13\n\x0bis_infected\x18\x05 \x01(\x08\x12\x13\n\x0bscan_result\x18\x06 \x01(\t\x12\x1a\n\x12scan_result_detail\x18\x07 \x01(\t2Y\n\x10VirusScanService\x12\x45\n\x08ScanFile\x12\x1a.virusscan.ScanFileRequest\x1a\x1b.virusscan.ScanFileResponse\"\x00\x62\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scan_pb2', _globals) -if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _globals['_SCANFILEREQUEST']._serialized_start=25 - _globals['_SCANFILEREQUEST']._serialized_end=119 - _globals['_SCANFILERESPONSE']._serialized_start=122 - _globals['_SCANFILERESPONSE']._serialized_end=287 - _globals['_VIRUSSCANSERVICE']._serialized_start=289 - _globals['_VIRUSSCANSERVICE']._serialized_end=378 -# @@protoc_insertion_point(module_scope) diff --git a/safeshare/safeshare_app/utils/virusRPC/scan_pb2.pyi b/safeshare/safeshare_app/utils/virusRPC/scan_pb2.pyi deleted file mode 100644 index 79c8c50..0000000 --- a/safeshare/safeshare_app/utils/virusRPC/scan_pb2.pyi +++ /dev/null @@ -1,35 +0,0 @@ -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Optional as _Optional - -DESCRIPTOR: _descriptor.FileDescriptor - -class ScanFileRequest(_message.Message): - __slots__ = ["file_name", "file_SHA256", "file_SHA1", "file_MD5"] - FILE_NAME_FIELD_NUMBER: _ClassVar[int] - FILE_SHA256_FIELD_NUMBER: _ClassVar[int] - FILE_SHA1_FIELD_NUMBER: _ClassVar[int] - FILE_MD5_FIELD_NUMBER: _ClassVar[int] - file_name: str - file_SHA256: str - file_SHA1: str - file_MD5: str - def __init__(self, file_name: _Optional[str] = ..., file_SHA256: _Optional[str] = ..., file_SHA1: _Optional[str] = ..., file_MD5: _Optional[str] = ...) -> None: ... - -class ScanFileResponse(_message.Message): - __slots__ = ["file_name", "file_SHA256", "file_SHA1", "file_MD5", "is_infected", "scan_result", "scan_result_detail"] - FILE_NAME_FIELD_NUMBER: _ClassVar[int] - FILE_SHA256_FIELD_NUMBER: _ClassVar[int] - FILE_SHA1_FIELD_NUMBER: _ClassVar[int] - FILE_MD5_FIELD_NUMBER: _ClassVar[int] - IS_INFECTED_FIELD_NUMBER: _ClassVar[int] - SCAN_RESULT_FIELD_NUMBER: _ClassVar[int] - SCAN_RESULT_DETAIL_FIELD_NUMBER: _ClassVar[int] - file_name: str - file_SHA256: str - file_SHA1: str - file_MD5: str - is_infected: bool - scan_result: str - scan_result_detail: str - def __init__(self, file_name: _Optional[str] = ..., file_SHA256: _Optional[str] = ..., file_SHA1: _Optional[str] = ..., file_MD5: _Optional[str] = ..., is_infected: bool = ..., scan_result: _Optional[str] = ..., scan_result_detail: _Optional[str] = ...) -> None: ... diff --git a/safeshare/safeshare_app/utils/virusRPC/scan_pb2_grpc.py b/safeshare/safeshare_app/utils/virusRPC/scan_pb2_grpc.py deleted file mode 100644 index 967a39e..0000000 --- a/safeshare/safeshare_app/utils/virusRPC/scan_pb2_grpc.py +++ /dev/null @@ -1,66 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc - -import scan_pb2 as scan__pb2 - - -class VirusScanServiceStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.ScanFile = channel.unary_unary( - '/virusscan.VirusScanService/ScanFile', - request_serializer=scan__pb2.ScanFileRequest.SerializeToString, - response_deserializer=scan__pb2.ScanFileResponse.FromString, - ) - - -class VirusScanServiceServicer(object): - """Missing associated documentation comment in .proto file.""" - - def ScanFile(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_VirusScanServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'ScanFile': grpc.unary_unary_rpc_method_handler( - servicer.ScanFile, - request_deserializer=scan__pb2.ScanFileRequest.FromString, - response_serializer=scan__pb2.ScanFileResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'virusscan.VirusScanService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class VirusScanService(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def ScanFile(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary(request, target, '/virusscan.VirusScanService/ScanFile', - scan__pb2.ScanFileRequest.SerializeToString, - scan__pb2.ScanFileResponse.FromString, - options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/safeshare/safeshare_app/utils/virusRPC/server.py b/safeshare/safeshare_app/utils/virusRPC/server.py deleted file mode 100644 index c7afc7e..0000000 --- a/safeshare/safeshare_app/utils/virusRPC/server.py +++ /dev/null @@ -1,61 +0,0 @@ -import grpc -import scan_pb2 -import scan_pb2_grpc -from concurrent import futures -import requests - -apiKey = "" # API key from VirusTotal - -headers = { - "accept": "application/json", - "x-apikey": apiKey -} - - -class VirusScanServicer(scan_pb2_grpc.VirusScanServiceServicer): - url = "https://www.virustotal.com/api/v3/files/" - - def ScanFile(self, request, context): - result = self.ScanSHA(request.file_SHA256) - return result - - def ScanSHA(self, sha256_hash: str) -> scan_pb2.ScanFileResponse: - self.url += sha256_hash - - response = requests.get(self.url, headers=headers) - data = response.json()["data"] - result = scan_pb2.ScanFileResponse() - - result.is_infected = data["attributes"]["last_analysis_stats"]["malicious"] > 0 - - result.file_name = data["attributes"]["names"][0] - result.file_SHA1 = data["attributes"]["sha1"] - result.file_SHA256 = data["attributes"]["sha256"] - - return result - - -class VirusScanServer: - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - scan_pb2_grpc.add_VirusScanServiceServicer_to_server(VirusScanServicer(), server) - server.add_insecure_port("[::]:50051") - - def serve(self): - try: - self.server.start() - print("Server started, listening on 50051") - self.server.wait_for_termination() - except KeyboardInterrupt: - print("Server stopped") - self.server.stop(0) - - def __del__(self): - self.server.stop(0) - - def __enter__(self): - return self - - -if __name__ == "__main__": - cal = VirusScanServer() - cal.serve() diff --git a/safeshare/safeshare_vdb/Dockerfile b/safeshare/safeshare_vdb/Dockerfile new file mode 100644 index 0000000..1c1e57f --- /dev/null +++ b/safeshare/safeshare_vdb/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11 +LABEL authors="junwei" + +# Copy the current directory contents into the container at /app +COPY . /app + +# Set the working directory to /app +WORKDIR /app + +# Install any needed packages for grpc and environ +RUN pip install grpcio grpcio-tools django-environ requests boto3 + +# Start Server at server.py +CMD ["python", "server.py"] \ No newline at end of file diff --git a/safeshare/start.sh b/safeshare/start.sh old mode 100644 new mode 100755 index d3c616b..654da13 --- a/safeshare/start.sh +++ b/safeshare/start.sh @@ -1,11 +1,16 @@ #!/bin/bash -# Wait until Database is ready -while ! nc -z db 3306; do sleep 1; done - +# Migrate the database (if needed) python manage.py migrate --noinput -# Seed the database with initial data -#python manage.py loaddata study_together_app/fixtures/* +# Start your Django server in the background +python manage.py runserver 0.0.0.0:8000 & -python manage.py runserver 0.0.0.0:8000 \ No newline at end of file +# Sleep briefly to allow the Django server to start (you can adjust the sleep duration as needed) +sleep 2 + +# Start the custom management command to run the trash collector +python manage.py start_trash_collector + +# Optionally, you can monitor the logs in real-time if needed +tail -f django_server.log trash_collector.log