From e89006a6c8eb7c2a63a5385cd175a0d7387c8864 Mon Sep 17 00:00:00 2001 From: Richie <2837357W@student.gla.ac.uk> Date: Mon, 23 Oct 2023 22:15:10 +0800 Subject: [PATCH] dynamo db to store hashes --- safeshare/requirements.txt | 3 + safeshare/safeshare_vdb/client.py | 26 ++++++ safeshare/safeshare_vdb/dynamo.proto | 16 ++++ safeshare/safeshare_vdb/dynamo_pb2.py | 29 +++++++ safeshare/safeshare_vdb/dynamo_pb2.pyi | 17 ++++ safeshare/safeshare_vdb/dynamo_pb2_grpc.py | 99 ++++++++++++++++++++++ safeshare/safeshare_vdb/server.py | 78 +++++++++++++++++ 7 files changed, 268 insertions(+) create mode 100644 safeshare/safeshare_vdb/client.py create mode 100644 safeshare/safeshare_vdb/dynamo.proto create mode 100644 safeshare/safeshare_vdb/dynamo_pb2.py create mode 100644 safeshare/safeshare_vdb/dynamo_pb2.pyi create mode 100644 safeshare/safeshare_vdb/dynamo_pb2_grpc.py create mode 100644 safeshare/safeshare_vdb/server.py diff --git a/safeshare/requirements.txt b/safeshare/requirements.txt index 5bf9d2d..6f28c30 100644 --- a/safeshare/requirements.txt +++ b/safeshare/requirements.txt @@ -2,6 +2,7 @@ annotated-types==0.5.0 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 asgiref==3.7.2 +boto3==1.28.68 certifi==2023.7.22 cffi==1.16.0 charset-normalizer==3.2.0 @@ -14,6 +15,8 @@ djangorestframework-simplejwt==5.3.0 dr-scaffold==2.1.2 drf-nested-routers==0.93.4 drf-yasg==1.21.7 +grpcio==1.59.0 +grpcio-tools==1.59.0 idna==3.4 inflect==7.0.0 inflection==0.5.1 diff --git a/safeshare/safeshare_vdb/client.py b/safeshare/safeshare_vdb/client.py new file mode 100644 index 0000000..034f657 --- /dev/null +++ b/safeshare/safeshare_vdb/client.py @@ -0,0 +1,26 @@ +from concurrent.futures import ThreadPoolExecutor + +import re +import grpc +import dynamo_pb2 as pb2 +import dynamo_pb2_grpc as pb2_grpc + +class Client: + def __init__(self): + self.channel = grpc.insecure_channel("localhost:50051") + self.stub = pb2_grpc.Dynamo_DBStub(self.channel) + + def CheckFile(self, sha_256_id: str): + response = self.stub.CheckFile(pb2.Request(file_hash=sha_256_id)) + print(response) + + def UpdateFile(self, sha_256_id: str): + response = self.stub.UpdateFile(pb2.Request(file_hash=sha_256_id)) + print(response) + + +if __name__ == "__main__": + client = Client() + id = "15e4313dddb45875ed67d1ab25f1f5b76f0b3a23e4fa9308c521e3fb30068028" + client.CheckFile(id) + client.UpdateFile(id) diff --git a/safeshare/safeshare_vdb/dynamo.proto b/safeshare/safeshare_vdb/dynamo.proto new file mode 100644 index 0000000..6a77b21 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package dynamo_db; + +service Dynamo_DB { + rpc CheckFile (Request) returns (Response) {} + rpc UpdateFile (Request) returns (Response) {} +} + +message Request { + string file_hash = 2; +} + +message Response { + bool is_exist = 1; +} \ No newline at end of file diff --git a/safeshare/safeshare_vdb/dynamo_pb2.py b/safeshare/safeshare_vdb/dynamo_pb2.py new file mode 100644 index 0000000..81254d5 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo_pb2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: dynamo.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x64ynamo.proto\x12\tdynamo_db\"\x1c\n\x07Request\x12\x11\n\tfile_hash\x18\x02 \x01(\t\"\x1c\n\x08Response\x12\x10\n\x08is_exist\x18\x01 \x01(\x08\x32|\n\tDynamo_DB\x12\x36\n\tCheckFile\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x12\x37\n\nUpdateFile\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dynamo_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_REQUEST']._serialized_start=27 + _globals['_REQUEST']._serialized_end=55 + _globals['_RESPONSE']._serialized_start=57 + _globals['_RESPONSE']._serialized_end=85 + _globals['_DYNAMO_DB']._serialized_start=87 + _globals['_DYNAMO_DB']._serialized_end=211 +# @@protoc_insertion_point(module_scope) diff --git a/safeshare/safeshare_vdb/dynamo_pb2.pyi b/safeshare/safeshare_vdb/dynamo_pb2.pyi new file mode 100644 index 0000000..8a69b81 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo_pb2.pyi @@ -0,0 +1,17 @@ +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class Request(_message.Message): + __slots__ = ["file_hash"] + FILE_HASH_FIELD_NUMBER: _ClassVar[int] + file_hash: str + def __init__(self, file_hash: _Optional[str] = ...) -> None: ... + +class Response(_message.Message): + __slots__ = ["is_exist"] + IS_EXIST_FIELD_NUMBER: _ClassVar[int] + is_exist: bool + def __init__(self, is_exist: bool = ...) -> None: ... diff --git a/safeshare/safeshare_vdb/dynamo_pb2_grpc.py b/safeshare/safeshare_vdb/dynamo_pb2_grpc.py new file mode 100644 index 0000000..4e2bab5 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo_pb2_grpc.py @@ -0,0 +1,99 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import dynamo_pb2 as dynamo__pb2 + + +class Dynamo_DBStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CheckFile = channel.unary_unary( + '/dynamo_db.Dynamo_DB/CheckFile', + request_serializer=dynamo__pb2.Request.SerializeToString, + response_deserializer=dynamo__pb2.Response.FromString, + ) + self.UpdateFile = channel.unary_unary( + '/dynamo_db.Dynamo_DB/UpdateFile', + request_serializer=dynamo__pb2.Request.SerializeToString, + response_deserializer=dynamo__pb2.Response.FromString, + ) + + +class Dynamo_DBServicer(object): + """Missing associated documentation comment in .proto file.""" + + def CheckFile(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UpdateFile(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_Dynamo_DBServicer_to_server(servicer, server): + rpc_method_handlers = { + 'CheckFile': grpc.unary_unary_rpc_method_handler( + servicer.CheckFile, + request_deserializer=dynamo__pb2.Request.FromString, + response_serializer=dynamo__pb2.Response.SerializeToString, + ), + 'UpdateFile': grpc.unary_unary_rpc_method_handler( + servicer.UpdateFile, + request_deserializer=dynamo__pb2.Request.FromString, + response_serializer=dynamo__pb2.Response.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'dynamo_db.Dynamo_DB', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Dynamo_DB(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def CheckFile(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/CheckFile', + dynamo__pb2.Request.SerializeToString, + dynamo__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def UpdateFile(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/UpdateFile', + dynamo__pb2.Request.SerializeToString, + dynamo__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/safeshare/safeshare_vdb/server.py b/safeshare/safeshare_vdb/server.py new file mode 100644 index 0000000..e16fd6d --- /dev/null +++ b/safeshare/safeshare_vdb/server.py @@ -0,0 +1,78 @@ +from concurrent.futures import ThreadPoolExecutor + +import re +import grpc +import dynamo_pb2 as pb2 +import dynamo_pb2_grpc as pb2_grpc + +import boto3 as boto # 1.28.68 + +# dynamo db instance +dynamodb = boto.resource('dynamodb') +sha256_table = dynamodb.Table('safeshare_sha256') +sha1_table = dynamodb.Table('safeshare_sha1') +md5_table = dynamodb.Table('safeshare_md5') + +# hash +hex_pattern = re.compile("^[a-fA-F0-9]+$") + + +def check_sha256(sha256): + response = sha256_table.get_item(Key={'sha256': sha256}) + return True if 'Item' in response else False + + +def check_sha1(sha1): + response = sha1_table.get_item(Key={'sha1': sha1}) + return True if 'Item' in response else False + + +def check_md5(md5): + response = md5_table.get_item(Key={'md5': md5}) + return True if 'Item' in response else False + + +class Dynamo(pb2_grpc.Dynamo_DBServicer): + def CheckFile(self, request, context): + if not hex_pattern.match(request.file_hash): + return pb2.Response(is_exist=False) + else: + length = len(request.file_hash) + if length == 64: + return pb2.Response(is_exist=check_sha256(request.file_hash)) + elif length == 40: + return pb2.Response(is_exist=check_sha1(request.file_hash)) + elif length == 32: + return pb2.Response(is_exist=check_md5(request.file_hash)) + else: + return pb2.Response(is_exist=False) + + def UpdateFile(self, request, context): + if not hex_pattern.match(request.file_hash): + return pb2.Response(is_exist=False) + else: + length = len(request.file_hash) + if length == 64: + sha256_table.put_item(Item={'sha256': request.file_hash}) + return pb2.Response(is_exist=True) + elif length == 40: + sha1_table.put_item(Item={'sha1': request.file_hash}) + return pb2.Response(is_exist=True) + elif length == 32: + md5_table.put_item(Item={'md5': request.file_hash}) + return pb2.Response(is_exist=True) + else: + return pb2.Response(is_exist=False) + + +def serve(): + server = grpc.server(ThreadPoolExecutor(max_workers=10)) + pb2_grpc.add_Dynamo_DBServicer_to_server(Dynamo(), server) + + server.add_insecure_port('[::]:50051') + server.start() + server.wait_for_termination() + + +if __name__ == '__main__': + serve()