Upgraded transmat protocol to ver. 0.2

README.md

@@ -1,21 +1,8 @@
 # Transphase
-This is a collection of tools to copy files or proxy connections via WebSockets protocol.
+This is a collection of tools to copy files or proxy connections via WebSockets protocol. \
-It is done in a cryptographically safe(ish) way.
+It is done in a cryptographically safe(ish) way. \
-A rendevouz/relay server is used to help with data transfer. 
+A relay server is used to help with data transfer. 
 
-**Transphase** - the relay server \
+**transphase** - the relay server \
-**Transmat** - tool to copy files/directories \
+**transmat** - tool to copy files/directories \
-**Transplace** - HTTP proxy/exit point 
+**transplace** - HTTP proxy/exit point 
-
-### Copy files
-Transphase server should be running and accessible by the parties. 
-On the sending party:
-```
-transmat --send <FILENAME>
-```
-This will prepare the sender and output the command to run on the receiving party, eg:
-```
-transmat --receive --password Space-Time-Continuum
-```
-
-See also ```transmat --help```

transmat/Readme.md

@@ -0,0 +1,17 @@
+### How to copy files using ```transmat```
+Transphase relay server should be running and accessible by the parties. 
+On the sending party:
+```
+transmat --send <FILENAME>
+```
+This will prepare the sender and output the command to run on the receiving party, eg:
+```
+transmat --receive --password Space-Time-Continuum
+```
+
+The sending party will wait for the receiving party to connect to the relay server, and then it will start the transfer. \
+With the exception of some service messages, all the data is encrypted. \
+Encryption and decryption is done only client-side, and the relay server has no useful knowledge of the data it relays. \
+Fernet module (AES128-CBC + HMAC-SHA256) is used to encrypt and authenticate the data.
+
+See also ```transmat --help```

transmat/requirements.txt

@@ -5,5 +5,4 @@ lz4==4.3.2
 progress==1.6
 pycparser==2.21
 pyjson==1.3.0
-websockets==11.0.2
+websockets==11.0.2
-xkcdpass==1.19.3

transmat/transmat.py

@@ -1,24 +1,32 @@
 #!/usr/bin/env python
 
 import asyncio, websockets
-import sys, os, base64, argparse, json, pickle
+import sys, os, base64, argparse, json, pickle, math
-from xkcdpass import xkcd_password as xp
 from cryptography.fernet import Fernet
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 from progress.bar import Bar
+import diceware
 
 
-VERSION_NUMBER="0.1"
+VERSION_NUMBER="0.2.0"
+PROTOCOL_VERSION="0.2"
+SCRIPT_BASENAME = os.path.basename(sys.argv[0])
+
+#async def read_chunks(filename, chunk_size = 1024):
+#    with open(filename, "rb") as f:
+#        while True:
+#            chunk = f.read(chunk_size)
+#            if not chunk:
+#                break
+#            yield chunk
 
 
-async def read_chunks(filename, chunk_size = 1024):
+def chunk_from_file(file, chunk_id, chunk_size):
-    with open(filename, "rb") as f:
+    position = chunk_id * chunk_size
-        while True:
+    file.seek(position)
-            chunk = f.read(chunk_size)
+    chunk = file.read(chunk_size)
-            if not chunk:
+    return chunk
-                break
-            yield chunk
 
 
 def derive_key_from_password(password):
@@ -68,9 +76,7 @@ async def send_msg(ws, msg):
 
 async def send_encrypted_msg(ws, k, data):
     (
-        msgtype, 
         peer_group_id, 
-        role,
         filename,
         chunk_size,
         chunk_id,
@@ -86,34 +92,29 @@ async def send_encrypted_msg(ws, k, data):
     "chunk": chunk
     }
     msg = {
-        "msgtype": msgtype,
         "peer_group_id": peer_group_id,
-        "role": role,
         "payload": encrypt_chunk(k, pickle.dumps(payload))
     }
     await send_msg(ws, json.dumps(msg))
 
 
-
 async def main():
-    WS_RELAY_SERVER = "wss://transmat.exocortex.ru"
+    WS_RELAY_SERVER = "wss://transmat.exocortex.ru/ws"
     parser = argparse.ArgumentParser()
     arg_group = parser.add_mutually_exclusive_group(required=True)
     arg_group.add_argument('--receive', '--recv', action='store_true', help='Receive a file from the remote party (mutually exclusive with --send and --relay)')
     arg_group.add_argument('--send', type=str, help='Send a file to the remote party (mutually exclusive with --receive and --relay)')
     arg_group.add_argument('--version', action='store_true', help="Show version")
-    #arg_group.add_argument('--relay', action='store_true', help='Run as a Relay server (mutually exclusive with --receive and --send)')
     parser.add_argument('--server', type=str, help="Specify the Relay server URL (ignored with --relay)")
     parser.add_argument('--password', type=str, help="Specify the shared password (for --receive)")
     args = parser.parse_args()    
     passwd_part = server_part = ""
-    send_part = f'{sys.argv[0]} --receive '
+    send_part = f'{SCRIPT_BASENAME} --receive'
     if args.receive:
         role = 'receive'
         password = args.password
     if args.send and args.password is None:
-        wordlist = xp.generate_wordlist(wordfile = xp.locate_wordfile(), min_length = 5, max_length = 9)
+        password = diceware.generate_passphrase(4)
-        password = xp.generate_xkcdpassword(wordlist, numwords=4, delimiter = "-", case='capitalize')
         passwd_part = f"--password {password}"
     if args.receive and args.password is None:
         print("Error: --password required when receiving files.")
@@ -128,71 +129,91 @@ async def main():
         WS_RELAY_SERVER = args.server
         server_part = f'--server {WS_RELAY_SERVER}'
     if args.version:
-        print(f"{sys.argv[0]} ver {VERSION_NUMBER}")
+        print(f"{SCRIPT_BASENAME} ver. {VERSION_NUMBER}, protocol ver. {PROTOCOL_VERSION}")
         sys.exit(0)
 
     k = derive_key_from_password(password)
     peer_group_id = get_peer_group_id(password)
 
     if role == 'send':
-        print('Run the following command on the remote party:\n\n', send_part, server_part, passwd_part, "\n")
+        print('Run the following command on the remote party:\n\n', send_part, passwd_part, server_part, "\n")
         filename = os.path.basename(file_path)
         file_size = os.path.getsize(file_path)
-        chunk_id = 0
+        file = open(file_path, 'rb')
         chunk_size = 1024 * 512
-        number_of_chunks = round(file_size / chunk_size)
+        chunk_id = ""
+        chunk = ""
+        number_of_chunks = math.ceil(file_size / chunk_size)
         WS_RELAY_SERVER = WS_RELAY_SERVER.replace('http', 'ws', 1)
         async with websockets.connect(WS_RELAY_SERVER) as ws:
-            msgtype = "announce"
+            # Announce ourself to the relay
-            await send_encrypted_msg(ws, k, (msgtype, peer_group_id, role, filename, "", "", number_of_chunks, ""))
+            await send_encrypted_msg(ws, k, (peer_group_id, filename, chunk_size, chunk_id, number_of_chunks, chunk))
+            bar = Bar(filename, max=number_of_chunks, check_tty=False, suffix='%(percent).1f%% complete - %(eta_td)s remaining')
             while True:
+                # Wait for messages from receiver
                 message = await ws.recv()
                 message = json.loads(message)
-                if message["msgtype"] == "announce" and message["peer_group_id"] == peer_group_id:
+                # If message is not meant for us - skip it
-                    break
+                if 'peer_group_id' not in message or message["peer_group_id"] != peer_group_id:
-            bar = Bar('Transferring', max=number_of_chunks,  suffix='%(percent).1f%% complete - %(eta_td)s remaining')
+                    pass
-            msgtype = "data"
+                payload = pickle.loads(decrypt_chunk(k, message["payload"]))
-            async for chunk in read_chunks(file_path, chunk_size):
+                chunk_id = payload["chunk_id"]
-                msg = (msgtype, peer_group_id, role, filename, chunk_size, chunk_id, number_of_chunks, chunk)
+                chunk = chunk_from_file(file, chunk_id, chunk_size)
+                #bar = Bar(filename, max=number_of_chunks, check_tty=False, suffix='%(percent).1f%% complete - %(eta_td)s remaining')
+                msg = (peer_group_id, filename, chunk_size, chunk_id, number_of_chunks, chunk)
                 await send_encrypted_msg(ws, k, msg)
-                await asyncio.sleep(0.05)
+                #bar.index = chunk_id
-                proceed = await ws.recv()
                 bar.next()
                 chunk_id += 1
-            bar.suffix = '%(percent).1f%% done'
+                if chunk_id > number_of_chunks:
-            print("\n")
+                    break    
+            
+            bar.suffix = '%(percent).1f%% complete'
+            bar.update()
+            print("")
 
     if role =='receive':
         async with websockets.connect(WS_RELAY_SERVER) as ws:
-            msgtype = "announce"
+            filename = ""
-            await send_encrypted_msg(ws, k, (msgtype, peer_group_id, role, "", "", "", "", ""))
+            chunk_size = ""
+            chunk_id = 0
+            number_of_chunks = ""
+            chunk = ""
             bar = None
             f = None
-            i = 1
             while True:
+                # Announce ourself to the relay
+                await send_encrypted_msg(ws, k, (peer_group_id, filename, chunk_size, chunk_id, number_of_chunks, chunk))
+                # Wait for message from sender
                 message = await ws.recv()
                 message = json.loads(message)
+                # If message is not meant for us - skip it
+                if 'peer_group_id' not in message or message["peer_group_id"] != peer_group_id:
+                    pass
                 payload = message["payload"]
                 msg = pickle.loads(decrypt_chunk(k, payload))
+                chunk_id = msg["chunk_id"]
+                number_of_chunks = msg["number_of_chunks"]
+                filename = msg["filename"]
                 if bar is None:
-                    filename = msg["filename"]
+                    bar = Bar(filename, max=number_of_chunks, check_tty=False, suffix='%(percent).1f%% complete - %(eta_td)s remaining')
-                    number_of_chunks = msg["number_of_chunks"]
-                    bar = Bar('Receiving', max=number_of_chunks,  suffix='%(percent).1f%% complete - %(eta_td)s remaining')
                 if f is None:
-                    f = open(msg["filename"], "wb")
+                    f = open(filename, "wb")
                     f.write(msg["chunk"])
                 else:
                     f.write(msg["chunk"])
-                msgtype = "proceed"
+                chunk_id += 1
-                await send_encrypted_msg(ws, k, (msgtype, peer_group_id, role, "", "", "", "", "")) # request the next chunk from sender
+                bar.suffix = "%(percent).1f%% complete - %(eta_td)s remaining"
-                i += 1
                 bar.next()
-                if i > msg["number_of_chunks"]:
+                if chunk_id >= number_of_chunks:
+                    # This was the last chunk, exit
+                    await send_encrypted_msg(ws, k, (peer_group_id, "", "", chunk_id, "", ""))
+                    f.close()
                     bar.suffix = '%(percent).1f%% complete'
                     bar.update()
-                    f.close()
+                    bar.finish()
-                    print("\n")
                     break
+ 
                 
 
 asyncio.run(main())

transphase.py

@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-import asyncio
-import websockets
-import logging 
-import json
-
-
-logging.basicConfig(
-    format="%(asctime)s %(message)s",
-    level=logging.INFO,
-)
-
-peer_list = {}
-
-class LoggerAdapter(logging.LoggerAdapter):
-    """Add connection ID and client IP address to websockets logs."""
-    def process(self, msg, kwargs):
-        try:
-            websocket = kwargs["extra"]["websocket"]
-        except KeyError:
-            return msg, kwargs
-        rip = websocket.remote_address[0]
-        try: 
-            xff = websocket.request_headers.get("X-Forwarded-For")
-        except:
-            xff = "None"
-        return f"{websocket.id} {rip} {xff}", kwargs
-
-async def handler(websocket):
-    peer_group_id = None
-    try:
-        while not websocket.closed:
-            message = await websocket.recv()
-            msg = json.loads(message)
-            if "peer_group_id" in msg:
-                peer_group_id = msg["peer_group_id"]
-                if peer_group_id not in peer_list:
-                    peer_list[peer_group_id] = set() # Create new set for all peers in peer_group_id
-                peer_list[peer_group_id].add(websocket) # Add peer's socket to peer_group
-            for peer in peer_list[peer_group_id]:
-                if peer != websocket:
-                    await peer.send(message)
-    except websockets.exceptions.ConnectionClosed as e:
-        pass
-    finally:
-        peer_list[peer_group_id].remove(websocket)
-        if len(peer_list[peer_group_id]) < 1:
-            peer_list.pop(peer_group_id)
-
-async def main():
-    async with websockets.serve(
-        handler, "", 8001, ping_timeout=30,
-        logger=LoggerAdapter(logging.getLogger("websockets.server"), None),
-        ):
-        await asyncio.Future()  # run forever
-
-if __name__ == "__main__":
-    asyncio.run(main())

transphase/docs/Message.md

@@ -0,0 +1,17 @@
+# Transmat message format
+## ver. 0.1
+
+### Message
+* msgtype (announce, data, confirmation)
+* per_group_id
+* role (send, receive)
+* payload (encrypted payload containing file chunks and service metadata)
+* *TODO - Add protocol version number*
+
+### Payload
+* filename (current filename without path)
+* chunk_size (size of each chunk)
+* chunk_id (number of current chunk)
+* numbr of chunks (total number of chunks in file)
+* chunk (current data)
+* *TODO - Add a hash of the chunk data to use in confirmation messages*

transphase/docs/Protocol.md

@@ -0,0 +1,22 @@
+# Transmat protocol description
+## ver. 0.1
+
+### **Dramatis Personae:**
+*Relay* - Transphase relay Relay
+*Sender* - sending party
+*Receiver* - receiving party
+
+* *Sender* creates a low-entropy password, derives a key and a `peer_group_id` vaule and sends the message containing the `peer_group_id` to the *Relay*
+* *Relay*  maintains a list of all its peers grouped by their `peer_group_id` value. 
+    * If the current message has a new unique `peer_group_id`, then a new peer group is created.
+    * If the current message has a known `peer_group_id`, then a new peer group is created from it and the party that sent the message (normally the *Sender*) is added to that group.
+* *Sender* passes the low-entropy password to the *Receiver* over a different channel (e.g. phone or SSH).
+* *Receiver* derives a key and a `peer_group_id` value and sends the message containing the `peer_group_id` to the *Relay*.
+    * This message contains 0 as `chunk_number` so that the *Sender* knows that the transfer should start from the very beginning.
+* *Relay* forwards this message to all other peers in the corresponding peer group (that is to the *Sender*).
+* *Sender* receives this message; sends the encrypted chunk and metadata; and then waits for the *Receiver* to send a confirmation that it has successfully received the  message. 
+* *Relay* forwards this message to all other peers in the corresponding peer group (that is to the *Receiver*).
+* *Receiver* decrypts the payload and writes it to the target file. (TODO: Check chunk hash) Then it sends a confirmation to the *Relay*. The confirmation contains the incremented number as the `chunk_id`
+    * If this was the last chunk in the transmission, *Receiver* closes the Websocket connection to the *Relay*
+* *Sender* receives the confirmation and sends the next encrypted chunk and metadata.
+    * If *Sender* has no more chunks to send, *Sender* closes the Websocket connection to the *Relay*

transphase/requirements.txt

@@ -0,0 +1,2 @@
+cryptography==40.0.2
+quart

transphase/transphase.py

@@ -0,0 +1,69 @@
+import json
+import asyncio
+import sys, time
+from quart import Quart, websocket
+
+peer_list = {}
+TRANSFERRED_DATA = 0
+
+app = Quart(__name__)
+
+
+def convert_bytes(byte_value):
+    if byte_value < 1024:  # Less than 1 kilobyte
+        return str(byte_value) +  " B"
+    elif byte_value < 1024**2:  # Less than 1 megabyte
+        kb_value = round(byte_value / 1024)
+        return str(kb_value) + " KB"
+    elif byte_value < 1024**3:  # Less than 1 gigabyte
+        mb_value = round(byte_value / (1024**2),2)
+        return str(mb_value) + " MB"
+    elif byte_value < 1024**4:  # Less than 1 terabyte
+        gb_value = round(byte_value / (1024**3),2)
+        return str(gb_value) + " GB"
+    elif byte_value < 1024**5:  # Less than 1 petabyte
+        tb_value = round(byte_value / (1024**4),2)
+        return str(tb_value) + " TB"
+    else:  # More than or equal to 1 petabyte
+        tb_value = round(byte_value / (1024**5),2)
+        return str(tb_value) + " PB"
+
+@app.route("/")
+async def retmain():
+    return f"Ready to relay\n"
+
+@app.websocket("/ws")
+async def handle_websockets():
+    global TRANSFERRED_DATA
+    peer_group_id = None
+    while True:
+        try:
+            message = await websocket.receive()
+            msg = json.loads(message)
+            if "peer_group_id" in msg:
+                peer_group_id = msg["peer_group_id"]
+                if peer_group_id not in peer_list:
+                    peer_list[peer_group_id] = set()
+                peer_list[peer_group_id].add(websocket._get_current_object())
+            for peer in peer_list[peer_group_id]:
+                if peer != websocket._get_current_object():
+                    await peer.send(message)
+                    TRANSFERRED_DATA = TRANSFERRED_DATA + sys.getsizeof(message)
+        except asyncio.exceptions.CancelledError:
+            peer_list[peer_group_id].remove(websocket._get_current_object())
+            if len(peer_list[peer_group_id]) < 1:
+                peer_list.pop(peer_group_id)
+
+@app.route("/stats")
+async def return_stats():
+    process_start_time = time.process_time()
+    current_time = time.time()
+    uptime = current_time - process_start_time
+    uptime_str = time.strftime("%H:%M:%S", time.gmtime(uptime))
+    peers_connected = 0
+    for p in peer_list.values():
+        peers_connected += len(p)
+    resp = f"<p><b>Uptime: </b>{uptime_str}</p><p><b>Peers: </b>{peers_connected}</p><p><b>Transferred: </b>{convert_bytes(TRANSFERRED_DATA)}</p>"
+    return resp
+
+app.run()