From e89006a6c8eb7c2a63a5385cd175a0d7387c8864 Mon Sep 17 00:00:00 2001 From: Richie <2837357W@student.gla.ac.uk> Date: Mon, 23 Oct 2023 22:15:10 +0800 Subject: [PATCH 1/3] dynamo db to store hashes --- safeshare/requirements.txt | 3 + safeshare/safeshare_vdb/client.py | 26 ++++++ safeshare/safeshare_vdb/dynamo.proto | 16 ++++ safeshare/safeshare_vdb/dynamo_pb2.py | 29 +++++++ safeshare/safeshare_vdb/dynamo_pb2.pyi | 17 ++++ safeshare/safeshare_vdb/dynamo_pb2_grpc.py | 99 ++++++++++++++++++++++ safeshare/safeshare_vdb/server.py | 78 +++++++++++++++++ 7 files changed, 268 insertions(+) create mode 100644 safeshare/safeshare_vdb/client.py create mode 100644 safeshare/safeshare_vdb/dynamo.proto create mode 100644 safeshare/safeshare_vdb/dynamo_pb2.py create mode 100644 safeshare/safeshare_vdb/dynamo_pb2.pyi create mode 100644 safeshare/safeshare_vdb/dynamo_pb2_grpc.py create mode 100644 safeshare/safeshare_vdb/server.py diff --git a/safeshare/requirements.txt b/safeshare/requirements.txt index 5bf9d2d..6f28c30 100644 --- a/safeshare/requirements.txt +++ b/safeshare/requirements.txt @@ -2,6 +2,7 @@ annotated-types==0.5.0 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 asgiref==3.7.2 +boto3==1.28.68 certifi==2023.7.22 cffi==1.16.0 charset-normalizer==3.2.0 @@ -14,6 +15,8 @@ djangorestframework-simplejwt==5.3.0 dr-scaffold==2.1.2 drf-nested-routers==0.93.4 drf-yasg==1.21.7 +grpcio==1.59.0 +grpcio-tools==1.59.0 idna==3.4 inflect==7.0.0 inflection==0.5.1 diff --git a/safeshare/safeshare_vdb/client.py b/safeshare/safeshare_vdb/client.py new file mode 100644 index 0000000..034f657 --- /dev/null +++ b/safeshare/safeshare_vdb/client.py @@ -0,0 +1,26 @@ +from concurrent.futures import ThreadPoolExecutor + +import re +import grpc +import dynamo_pb2 as pb2 +import dynamo_pb2_grpc as pb2_grpc + +class Client: + def __init__(self): + self.channel = grpc.insecure_channel("localhost:50051") + self.stub = pb2_grpc.Dynamo_DBStub(self.channel) + + def CheckFile(self, sha_256_id: str): + response = self.stub.CheckFile(pb2.Request(file_hash=sha_256_id)) + print(response) + + def UpdateFile(self, sha_256_id: str): + response = self.stub.UpdateFile(pb2.Request(file_hash=sha_256_id)) + print(response) + + +if __name__ == "__main__": + client = Client() + id = "15e4313dddb45875ed67d1ab25f1f5b76f0b3a23e4fa9308c521e3fb30068028" + client.CheckFile(id) + client.UpdateFile(id) diff --git a/safeshare/safeshare_vdb/dynamo.proto b/safeshare/safeshare_vdb/dynamo.proto new file mode 100644 index 0000000..6a77b21 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package dynamo_db; + +service Dynamo_DB { + rpc CheckFile (Request) returns (Response) {} + rpc UpdateFile (Request) returns (Response) {} +} + +message Request { + string file_hash = 2; +} + +message Response { + bool is_exist = 1; +} \ No newline at end of file diff --git a/safeshare/safeshare_vdb/dynamo_pb2.py b/safeshare/safeshare_vdb/dynamo_pb2.py new file mode 100644 index 0000000..81254d5 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo_pb2.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: dynamo.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x64ynamo.proto\x12\tdynamo_db\"\x1c\n\x07Request\x12\x11\n\tfile_hash\x18\x02 \x01(\t\"\x1c\n\x08Response\x12\x10\n\x08is_exist\x18\x01 \x01(\x08\x32|\n\tDynamo_DB\x12\x36\n\tCheckFile\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x12\x37\n\nUpdateFile\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dynamo_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_REQUEST']._serialized_start=27 + _globals['_REQUEST']._serialized_end=55 + _globals['_RESPONSE']._serialized_start=57 + _globals['_RESPONSE']._serialized_end=85 + _globals['_DYNAMO_DB']._serialized_start=87 + _globals['_DYNAMO_DB']._serialized_end=211 +# @@protoc_insertion_point(module_scope) diff --git a/safeshare/safeshare_vdb/dynamo_pb2.pyi b/safeshare/safeshare_vdb/dynamo_pb2.pyi new file mode 100644 index 0000000..8a69b81 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo_pb2.pyi @@ -0,0 +1,17 @@ +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Optional as _Optional + +DESCRIPTOR: _descriptor.FileDescriptor + +class Request(_message.Message): + __slots__ = ["file_hash"] + FILE_HASH_FIELD_NUMBER: _ClassVar[int] + file_hash: str + def __init__(self, file_hash: _Optional[str] = ...) -> None: ... + +class Response(_message.Message): + __slots__ = ["is_exist"] + IS_EXIST_FIELD_NUMBER: _ClassVar[int] + is_exist: bool + def __init__(self, is_exist: bool = ...) -> None: ... diff --git a/safeshare/safeshare_vdb/dynamo_pb2_grpc.py b/safeshare/safeshare_vdb/dynamo_pb2_grpc.py new file mode 100644 index 0000000..4e2bab5 --- /dev/null +++ b/safeshare/safeshare_vdb/dynamo_pb2_grpc.py @@ -0,0 +1,99 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import dynamo_pb2 as dynamo__pb2 + + +class Dynamo_DBStub(object): + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.CheckFile = channel.unary_unary( + '/dynamo_db.Dynamo_DB/CheckFile', + request_serializer=dynamo__pb2.Request.SerializeToString, + response_deserializer=dynamo__pb2.Response.FromString, + ) + self.UpdateFile = channel.unary_unary( + '/dynamo_db.Dynamo_DB/UpdateFile', + request_serializer=dynamo__pb2.Request.SerializeToString, + response_deserializer=dynamo__pb2.Response.FromString, + ) + + +class Dynamo_DBServicer(object): + """Missing associated documentation comment in .proto file.""" + + def CheckFile(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def UpdateFile(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_Dynamo_DBServicer_to_server(servicer, server): + rpc_method_handlers = { + 'CheckFile': grpc.unary_unary_rpc_method_handler( + servicer.CheckFile, + request_deserializer=dynamo__pb2.Request.FromString, + response_serializer=dynamo__pb2.Response.SerializeToString, + ), + 'UpdateFile': grpc.unary_unary_rpc_method_handler( + servicer.UpdateFile, + request_deserializer=dynamo__pb2.Request.FromString, + response_serializer=dynamo__pb2.Response.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'dynamo_db.Dynamo_DB', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Dynamo_DB(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def CheckFile(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/CheckFile', + dynamo__pb2.Request.SerializeToString, + dynamo__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def UpdateFile(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/UpdateFile', + dynamo__pb2.Request.SerializeToString, + dynamo__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/safeshare/safeshare_vdb/server.py b/safeshare/safeshare_vdb/server.py new file mode 100644 index 0000000..e16fd6d --- /dev/null +++ b/safeshare/safeshare_vdb/server.py @@ -0,0 +1,78 @@ +from concurrent.futures import ThreadPoolExecutor + +import re +import grpc +import dynamo_pb2 as pb2 +import dynamo_pb2_grpc as pb2_grpc + +import boto3 as boto # 1.28.68 + +# dynamo db instance +dynamodb = boto.resource('dynamodb') +sha256_table = dynamodb.Table('safeshare_sha256') +sha1_table = dynamodb.Table('safeshare_sha1') +md5_table = dynamodb.Table('safeshare_md5') + +# hash +hex_pattern = re.compile("^[a-fA-F0-9]+$") + + +def check_sha256(sha256): + response = sha256_table.get_item(Key={'sha256': sha256}) + return True if 'Item' in response else False + + +def check_sha1(sha1): + response = sha1_table.get_item(Key={'sha1': sha1}) + return True if 'Item' in response else False + + +def check_md5(md5): + response = md5_table.get_item(Key={'md5': md5}) + return True if 'Item' in response else False + + +class Dynamo(pb2_grpc.Dynamo_DBServicer): + def CheckFile(self, request, context): + if not hex_pattern.match(request.file_hash): + return pb2.Response(is_exist=False) + else: + length = len(request.file_hash) + if length == 64: + return pb2.Response(is_exist=check_sha256(request.file_hash)) + elif length == 40: + return pb2.Response(is_exist=check_sha1(request.file_hash)) + elif length == 32: + return pb2.Response(is_exist=check_md5(request.file_hash)) + else: + return pb2.Response(is_exist=False) + + def UpdateFile(self, request, context): + if not hex_pattern.match(request.file_hash): + return pb2.Response(is_exist=False) + else: + length = len(request.file_hash) + if length == 64: + sha256_table.put_item(Item={'sha256': request.file_hash}) + return pb2.Response(is_exist=True) + elif length == 40: + sha1_table.put_item(Item={'sha1': request.file_hash}) + return pb2.Response(is_exist=True) + elif length == 32: + md5_table.put_item(Item={'md5': request.file_hash}) + return pb2.Response(is_exist=True) + else: + return pb2.Response(is_exist=False) + + +def serve(): + server = grpc.server(ThreadPoolExecutor(max_workers=10)) + pb2_grpc.add_Dynamo_DBServicer_to_server(Dynamo(), server) + + server.add_insecure_port('[::]:50051') + server.start() + server.wait_for_termination() + + +if __name__ == '__main__': + serve() From 16a00f0b960e8d9966e6e69bf9fbdc8f4e3ad710 Mon Sep 17 00:00:00 2001 From: Richie <2837357W@student.gla.ac.uk> Date: Mon, 23 Oct 2023 22:18:35 +0800 Subject: [PATCH 2/3] dynamo db to store hashes --- safeshare/safeshare_vdb/client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/safeshare/safeshare_vdb/client.py b/safeshare/safeshare_vdb/client.py index 034f657..0c3b5f5 100644 --- a/safeshare/safeshare_vdb/client.py +++ b/safeshare/safeshare_vdb/client.py @@ -1,6 +1,3 @@ -from concurrent.futures import ThreadPoolExecutor - -import re import grpc import dynamo_pb2 as pb2 import dynamo_pb2_grpc as pb2_grpc From 8eaad6659e3bca3c5d3f91297296970c4460a3ef Mon Sep 17 00:00:00 2001 From: Richie <2837357W@student.gla.ac.uk> Date: Mon, 23 Oct 2023 22:25:15 +0800 Subject: [PATCH 3/3] modify return and function names --- safeshare/safeshare_vdb/dynamo.proto | 4 ++-- safeshare/safeshare_vdb/dynamo_pb2.py | 2 +- safeshare/safeshare_vdb/dynamo_pb2_grpc.py | 28 +++++++++++----------- safeshare/safeshare_vdb/server.py | 10 ++++---- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/safeshare/safeshare_vdb/dynamo.proto b/safeshare/safeshare_vdb/dynamo.proto index 6a77b21..5bcfbb4 100644 --- a/safeshare/safeshare_vdb/dynamo.proto +++ b/safeshare/safeshare_vdb/dynamo.proto @@ -3,8 +3,8 @@ syntax = "proto3"; package dynamo_db; service Dynamo_DB { - rpc CheckFile (Request) returns (Response) {} - rpc UpdateFile (Request) returns (Response) {} + rpc CheckHash (Request) returns (Response) {} + rpc UpdateHash (Request) returns (Response) {} } message Request { diff --git a/safeshare/safeshare_vdb/dynamo_pb2.py b/safeshare/safeshare_vdb/dynamo_pb2.py index 81254d5..95b2918 100644 --- a/safeshare/safeshare_vdb/dynamo_pb2.py +++ b/safeshare/safeshare_vdb/dynamo_pb2.py @@ -13,7 +13,7 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x64ynamo.proto\x12\tdynamo_db\"\x1c\n\x07Request\x12\x11\n\tfile_hash\x18\x02 \x01(\t\"\x1c\n\x08Response\x12\x10\n\x08is_exist\x18\x01 \x01(\x08\x32|\n\tDynamo_DB\x12\x36\n\tCheckFile\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x12\x37\n\nUpdateFile\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x64ynamo.proto\x12\tdynamo_db\"\x1c\n\x07Request\x12\x11\n\tfile_hash\x18\x02 \x01(\t\"\x1c\n\x08Response\x12\x10\n\x08is_exist\x18\x01 \x01(\x08\x32|\n\tDynamo_DB\x12\x36\n\tCheckHash\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x12\x37\n\nUpdateHash\x12\x12.dynamo_db.Request\x1a\x13.dynamo_db.Response\"\x00\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) diff --git a/safeshare/safeshare_vdb/dynamo_pb2_grpc.py b/safeshare/safeshare_vdb/dynamo_pb2_grpc.py index 4e2bab5..f062266 100644 --- a/safeshare/safeshare_vdb/dynamo_pb2_grpc.py +++ b/safeshare/safeshare_vdb/dynamo_pb2_grpc.py @@ -14,13 +14,13 @@ class Dynamo_DBStub(object): Args: channel: A grpc.Channel. """ - self.CheckFile = channel.unary_unary( - '/dynamo_db.Dynamo_DB/CheckFile', + self.CheckHash = channel.unary_unary( + '/dynamo_db.Dynamo_DB/CheckHash', request_serializer=dynamo__pb2.Request.SerializeToString, response_deserializer=dynamo__pb2.Response.FromString, ) - self.UpdateFile = channel.unary_unary( - '/dynamo_db.Dynamo_DB/UpdateFile', + self.UpdateHash = channel.unary_unary( + '/dynamo_db.Dynamo_DB/UpdateHash', request_serializer=dynamo__pb2.Request.SerializeToString, response_deserializer=dynamo__pb2.Response.FromString, ) @@ -29,13 +29,13 @@ class Dynamo_DBStub(object): class Dynamo_DBServicer(object): """Missing associated documentation comment in .proto file.""" - def CheckFile(self, request, context): + def CheckHash(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def UpdateFile(self, request, context): + def UpdateHash(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -44,13 +44,13 @@ class Dynamo_DBServicer(object): def add_Dynamo_DBServicer_to_server(servicer, server): rpc_method_handlers = { - 'CheckFile': grpc.unary_unary_rpc_method_handler( - servicer.CheckFile, + 'CheckHash': grpc.unary_unary_rpc_method_handler( + servicer.CheckHash, request_deserializer=dynamo__pb2.Request.FromString, response_serializer=dynamo__pb2.Response.SerializeToString, ), - 'UpdateFile': grpc.unary_unary_rpc_method_handler( - servicer.UpdateFile, + 'UpdateHash': grpc.unary_unary_rpc_method_handler( + servicer.UpdateHash, request_deserializer=dynamo__pb2.Request.FromString, response_serializer=dynamo__pb2.Response.SerializeToString, ), @@ -65,7 +65,7 @@ class Dynamo_DB(object): """Missing associated documentation comment in .proto file.""" @staticmethod - def CheckFile(request, + def CheckHash(request, target, options=(), channel_credentials=None, @@ -75,14 +75,14 @@ class Dynamo_DB(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/CheckFile', + return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/CheckHash', dynamo__pb2.Request.SerializeToString, dynamo__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def UpdateFile(request, + def UpdateHash(request, target, options=(), channel_credentials=None, @@ -92,7 +92,7 @@ class Dynamo_DB(object): wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/UpdateFile', + return grpc.experimental.unary_unary(request, target, '/dynamo_db.Dynamo_DB/UpdateHash', dynamo__pb2.Request.SerializeToString, dynamo__pb2.Response.FromString, options, channel_credentials, diff --git a/safeshare/safeshare_vdb/server.py b/safeshare/safeshare_vdb/server.py index e16fd6d..e8a875a 100644 --- a/safeshare/safeshare_vdb/server.py +++ b/safeshare/safeshare_vdb/server.py @@ -19,21 +19,21 @@ hex_pattern = re.compile("^[a-fA-F0-9]+$") def check_sha256(sha256): response = sha256_table.get_item(Key={'sha256': sha256}) - return True if 'Item' in response else False + return 'Item' in response def check_sha1(sha1): response = sha1_table.get_item(Key={'sha1': sha1}) - return True if 'Item' in response else False + return 'Item' in response def check_md5(md5): response = md5_table.get_item(Key={'md5': md5}) - return True if 'Item' in response else False + return 'Item' in response class Dynamo(pb2_grpc.Dynamo_DBServicer): - def CheckFile(self, request, context): + def CheckHash(self, request, context): if not hex_pattern.match(request.file_hash): return pb2.Response(is_exist=False) else: @@ -47,7 +47,7 @@ class Dynamo(pb2_grpc.Dynamo_DBServicer): else: return pb2.Response(is_exist=False) - def UpdateFile(self, request, context): + def UpdateHash(self, request, context): if not hex_pattern.match(request.file_hash): return pb2.Response(is_exist=False) else: