devoalda
/
Safe_Share
mirror of https://github.com/Devoalda/Safe_Share.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Docker integration (#4) 

* fix(Docker Integration):

Cleaned unused files
Updated Trash Collector
Updated Docker Compose
Updated Start Script

* feature(microservice rpc docker):

* feature(microservice rpc docker):

* fix(Added API Environment):

Added environment variable for docker compose
This commit is contained in:
Devoalda 2023-10-30 08:22:11 +08:00 committed by GitHub
parent 52fe75d916
commit e534fa4e13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 84 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
safeshare/docker-compose.yml
View File

@ -1,54 +1,35 @@
version: '3'
services:
db:
image: mariadb:10.5
command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW
restart: always
volumes:
- db_data:/var/lib/mysql
networks:
- dbnet
environment:
- MYSQL_DATABASE=safeShare
- MYSQL_USER=user
- MYSQL_ROOT_PASSWORD=password
- MYSQL_PASSWORD=password
expose:
- 3306
redis:
image: redis:latest
command: redis-server --requirepass password
command: redis-server
restart: always
networks:
- dbnet
expose:
- 6379
volumes:
- redis_data:/data
- redis_data:/data
safeshare:
build:
context: .
dockerfile: Dockerfile
restart:
always
always
ports:
- "8000:8000" # Map container port 8000 to host port 8000
environment:
- DEBUG=True
- SECRET_KEY=A_RANDOM_SECRET_KEY
- DB_NAME=testSite
- DB_USER=user
- DB_PASSWORD=password
- DB_HOST=db
- DB_PORT=3306
- ALLOWED_HOSTS=*
- CACHE=True
- REDIS_URL=redis://:password@redis:6379/0
- REDIS_HOST=redis
- REDIS_PORT=6379
- REDIS_DB=0
- TRASH_TIMEOUT=60
depends_on:
- db
- redis
networks:
- dbnet
@ -58,16 +39,28 @@ services:
context: ./safeshare-frontend
dockerfile: Dockerfile
restart:
always
always
ports:
- "3000:3000" # Map container port 3000 to host port 3000
environment:
- REACT_APP_API_URL=http://localhost:8000/api
networks:
- dbnet
volumes:
db_data:
redis_data:
safeshare-virus:
build:
context: ./safeshare_vdb
dockerfile: Dockerfile
restart:
always
ports:
- "50051:50051"
environment:
- API_TOKEN=YOUR_API_TOKEN
networks:
- dbnet
volumes:
redis_data:
networks:
dbnet:

17
safeshare/safeshare/settings.py
View File

@ -39,6 +39,7 @@ ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*'])
REDIS_HOST = env.str('REDIS_HOST', default='localhost')
REDIS_PORT = env.str('REDIS_PORT', default='6379')
REDIS_DB = env.str('REDIS_DB', default='0')
TRASH_TIMEOUT = env.int('TRASH_TIMEOUT', default=60)
# Application definition
@ -136,15 +137,13 @@ if os.getenv('GITHUB_WORKFLOW'):
else:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.mysql',
'NAME': env.str('DB_NAME', 'safeshare'),
'USER': env.str('DB_USER', 'mariadb'),
'PASSWORD': env.str('DB_PASSWORD', 'mariadb'),
'HOST': env.str('DB_HOST', 'localhost'),
'PORT': env.str('DB_PORT', '3306'),
'OPTIONS': {
'init_command': "SET sql_mode='STRICT_TRANS_TABLES', innodb_strict_mode=1",
},
'ENGINE': 'django.db.backends.sqlite3',
# Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), # Or path to database file if using sqlite3.
'USER': '', # Not used with sqlite3.
'PASSWORD': '', # Not used with sqlite3.
'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
'PORT': '', # Set to empty string for default. Not used with sqlite3.
}
}

3
safeshare/safeshare/urls.py
View File

@ -17,9 +17,8 @@ Including another URLconf
from django.contrib import admin
from django.urls import path, include
from safeshare_app.utils.TrashCollector import TrashCollector
import threading
trash_collector = TrashCollector()
trash_collector.start()
urlpatterns = [
path('admin/', admin.site.urls),

0
safeshare/safeshare_app/utils/virusRPC/__init__.py → safeshare/safeshare_app/management/commands/__init__.py
View File

9
safeshare/safeshare_app/management/commands/start_trash_collector.py Normal file
View File

@ -0,0 +1,9 @@
from django.core.management.base import BaseCommand
from safeshare_app.utils.TrashCollector.TrashCollector import TrashCollector
class Command(BaseCommand):
help = 'Start the trash collector'
def handle(self, *args, **options):
trash_collector = TrashCollector()
trash_collector.start()

26
safeshare/safeshare_app/utils/TrashCollector.py → safeshare/safeshare_app/utils/TrashCollector/TrashCollector.py
View File

@ -1,7 +1,8 @@
import threading
import os
import redis
import threading
import environ
import redis
from django.conf import settings
@ -11,13 +12,11 @@ class TrashCollector:
self.thread = threading.Thread(target=self.run)
self.media_root = settings.MEDIA_ROOT
environ.Env.read_env(os.path.join('safeshare', 'env'))
self.env = environ.Env()
# Connect to Redis
self.redis = redis.StrictRedis(
host=self.env.str('REDIS_HOST', default='localhost'),
port=self.env.str('REDIS_PORT', default=6379),
db=self.env.str('REDIS_DB', default=0),
host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=settings.REDIS_DB,
)
def start(self):
@ -67,4 +66,15 @@ class TrashCollector:
print(e)
# Sleep for a specific interval in seconds
self.stop_event.wait(timeout=self.env.int('TRASH_TIMEOUT', default=60))
self.stop_event.wait(timeout=settings.TRASH_TIMEOUT)
if __name__ == '__main__':
trash_collector = TrashCollector()
try:
print("Starting trash collector")
trash_collector.start()
print("Trash collector started")
except KeyboardInterrupt:
trash_collector.stop()
print("Trash collector stopped")

20
safeshare/safeshare_app/utils/virusRPC/client.py
View File

@ -1,20 +0,0 @@
import grpc
import scan_pb2_grpc
import scan_pb2
class Client:
def __init__(self):
self.channel = grpc.insecure_channel("localhost:50051")
self.stub = scan_pb2_grpc.VirusScanServiceStub(self.channel)
def ScanFile(self, sha_256_id: str):
response = self.stub.ScanFile(scan_pb2.ScanFileRequest(file_SHA256=sha_256_id))
print(response)
if __name__ == "__main__":
client = Client()
id = "15e4313dddb45875ed67d1ab25f1f5b76f0b3a23e4fa9308c521e3fb30068028"
client.ScanFile(id)

24
safeshare/safeshare_app/utils/virusRPC/scan.proto
View File

@ -1,24 +0,0 @@
syntax = "proto3";
package virusscan;
service VirusScanService {
rpc ScanFile (ScanFileRequest) returns (ScanFileResponse) {}
}
message ScanFileRequest {
string file_name = 1;
string file_SHA256 = 2;
string file_SHA1 = 3;
string file_MD5 = 4;
}
message ScanFileResponse {
string file_name = 1;
string file_SHA256 = 2;
string file_SHA1 = 3;
string file_MD5 = 4;
bool is_infected = 5;
string scan_result = 6;
string scan_result_detail = 7;
}

29
safeshare/safeshare_app/utils/virusRPC/scan_pb2.py
View File

@ -1,29 +0,0 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: scan.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nscan.proto\x12\tvirusscan\"^\n\x0fScanFileRequest\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ile_SHA256\x18\x02 \x01(\t\x12\x11\n\tfile_SHA1\x18\x03 \x01(\t\x12\x10\n\x08\x66ile_MD5\x18\x04 \x01(\t\"\xa5\x01\n\x10ScanFileResponse\x12\x11\n\tfile_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66ile_SHA256\x18\x02 \x01(\t\x12\x11\n\tfile_SHA1\x18\x03 \x01(\t\x12\x10\n\x08\x66ile_MD5\x18\x04 \x01(\t\x12\x13\n\x0bis_infected\x18\x05 \x01(\x08\x12\x13\n\x0bscan_result\x18\x06 \x01(\t\x12\x1a\n\x12scan_result_detail\x18\x07 \x01(\t2Y\n\x10VirusScanService\x12\x45\n\x08ScanFile\x12\x1a.virusscan.ScanFileRequest\x1a\x1b.virusscan.ScanFileResponse\"\x00\x62\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'scan_pb2', _globals)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_globals['_SCANFILEREQUEST']._serialized_start=25
_globals['_SCANFILEREQUEST']._serialized_end=119
_globals['_SCANFILERESPONSE']._serialized_start=122
_globals['_SCANFILERESPONSE']._serialized_end=287
_globals['_VIRUSSCANSERVICE']._serialized_start=289
_globals['_VIRUSSCANSERVICE']._serialized_end=378
# @@protoc_insertion_point(module_scope)

35
safeshare/safeshare_app/utils/virusRPC/scan_pb2.pyi
View File

@ -1,35 +0,0 @@
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from typing import ClassVar as _ClassVar, Optional as _Optional
DESCRIPTOR: _descriptor.FileDescriptor
class ScanFileRequest(_message.Message):
__slots__ = ["file_name", "file_SHA256", "file_SHA1", "file_MD5"]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_SHA256_FIELD_NUMBER: _ClassVar[int]
FILE_SHA1_FIELD_NUMBER: _ClassVar[int]
FILE_MD5_FIELD_NUMBER: _ClassVar[int]
file_name: str
file_SHA256: str
file_SHA1: str
file_MD5: str
def __init__(self, file_name: _Optional[str] = ..., file_SHA256: _Optional[str] = ..., file_SHA1: _Optional[str] = ..., file_MD5: _Optional[str] = ...) -> None: ...
class ScanFileResponse(_message.Message):
__slots__ = ["file_name", "file_SHA256", "file_SHA1", "file_MD5", "is_infected", "scan_result", "scan_result_detail"]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_SHA256_FIELD_NUMBER: _ClassVar[int]
FILE_SHA1_FIELD_NUMBER: _ClassVar[int]
FILE_MD5_FIELD_NUMBER: _ClassVar[int]
IS_INFECTED_FIELD_NUMBER: _ClassVar[int]
SCAN_RESULT_FIELD_NUMBER: _ClassVar[int]
SCAN_RESULT_DETAIL_FIELD_NUMBER: _ClassVar[int]
file_name: str
file_SHA256: str
file_SHA1: str
file_MD5: str
is_infected: bool
scan_result: str
scan_result_detail: str
def __init__(self, file_name: _Optional[str] = ..., file_SHA256: _Optional[str] = ..., file_SHA1: _Optional[str] = ..., file_MD5: _Optional[str] = ..., is_infected: bool = ..., scan_result: _Optional[str] = ..., scan_result_detail: _Optional[str] = ...) -> None: ...

66
safeshare/safeshare_app/utils/virusRPC/scan_pb2_grpc.py
View File

@ -1,66 +0,0 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import scan_pb2 as scan__pb2
class VirusScanServiceStub(object):
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.ScanFile = channel.unary_unary(
'/virusscan.VirusScanService/ScanFile',
request_serializer=scan__pb2.ScanFileRequest.SerializeToString,
response_deserializer=scan__pb2.ScanFileResponse.FromString,
)
class VirusScanServiceServicer(object):
"""Missing associated documentation comment in .proto file."""
def ScanFile(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_VirusScanServiceServicer_to_server(servicer, server):
rpc_method_handlers = {
'ScanFile': grpc.unary_unary_rpc_method_handler(
servicer.ScanFile,
request_deserializer=scan__pb2.ScanFileRequest.FromString,
response_serializer=scan__pb2.ScanFileResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'virusscan.VirusScanService', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class VirusScanService(object):
"""Missing associated documentation comment in .proto file."""
@staticmethod
def ScanFile(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/virusscan.VirusScanService/ScanFile',
scan__pb2.ScanFileRequest.SerializeToString,
scan__pb2.ScanFileResponse.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

61
safeshare/safeshare_app/utils/virusRPC/server.py
View File

@ -1,61 +0,0 @@
import grpc
import scan_pb2
import scan_pb2_grpc
from concurrent import futures
import requests
apiKey = "" # API key from VirusTotal
headers = {
"accept": "application/json",
"x-apikey": apiKey
}
class VirusScanServicer(scan_pb2_grpc.VirusScanServiceServicer):
url = "https://www.virustotal.com/api/v3/files/"
def ScanFile(self, request, context):
result = self.ScanSHA(request.file_SHA256)
return result
def ScanSHA(self, sha256_hash: str) -> scan_pb2.ScanFileResponse:
self.url += sha256_hash
response = requests.get(self.url, headers=headers)
data = response.json()["data"]
result = scan_pb2.ScanFileResponse()
result.is_infected = data["attributes"]["last_analysis_stats"]["malicious"] > 0
result.file_name = data["attributes"]["names"][0]
result.file_SHA1 = data["attributes"]["sha1"]
result.file_SHA256 = data["attributes"]["sha256"]
return result
class VirusScanServer:
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
scan_pb2_grpc.add_VirusScanServiceServicer_to_server(VirusScanServicer(), server)
server.add_insecure_port("[::]:50051")
def serve(self):
try:
self.server.start()
print("Server started, listening on 50051")
self.server.wait_for_termination()
except KeyboardInterrupt:
print("Server stopped")
self.server.stop(0)
def __del__(self):
self.server.stop(0)
def __enter__(self):
return self
if __name__ == "__main__":
cal = VirusScanServer()
cal.serve()

14
safeshare/safeshare_vdb/Dockerfile Normal file
View File

@ -0,0 +1,14 @@
FROM python:3.11
LABEL authors="junwei"
# Copy the current directory contents into the container at /app
COPY . /app
# Set the working directory to /app
WORKDIR /app
# Install any needed packages for grpc and environ
RUN pip install grpcio grpcio-tools django-environ requests boto3
# Start Server at server.py
CMD ["python", "server.py"]

17
safeshare/start.sh Normal file → Executable file
View File

@ -1,11 +1,16 @@
#!/bin/bash
# Wait until Database is ready
while ! nc -z db 3306; do sleep 1; done
# Migrate the database (if needed)
python manage.py migrate --noinput
# Seed the database with initial data
#python manage.py loaddata study_together_app/fixtures/*
# Start your Django server in the background
python manage.py runserver 0.0.0.0:8000 &
python manage.py runserver 0.0.0.0:8000
# Sleep briefly to allow the Django server to start (you can adjust the sleep duration as needed)
sleep 2
# Start the custom management command to run the trash collector
python manage.py start_trash_collector
# Optionally, you can monitor the logs in real-time if needed
tail -f django_server.log trash_collector.log