Documentation for Transmat

README.md

@@ -5,22 +5,4 @@ A relay server is used to help with data transfer.
 
 **transphase** - the relay server \
 **transmat** - tool to copy files/directories \
-**transplace** - HTTP proxy/exit point 
-
-### How to copy files using ```transmat```
-Transphase relay server should be running and accessible by the parties. 
-On the sending party:
-```
-transmat --send <FILENAME>
-```
-This will prepare the sender and output the command to run on the receiving party, eg:
-```
-transmat --receive --password Space-Time-Continuum
-```
-
-The sending party will wait for the receiving party to connect to the relay server, and then it will start the transfer. \
-With the exception of some service messages, all the data is encrypted. \
-Encryption and decryption is done only client-side, and the relay server has no useful knowledge of the data it relays. \
-Fernet module (AES128-CBC + HMAC-SHA256) is used to encrypt and authenticate the data.
-
-See also ```transmat --help```
+**transplace** - HTTP proxy/exit point 

transmat/Readme.md

@@ -0,0 +1,17 @@
+### How to copy files using ```transmat```
+Transphase relay server should be running and accessible by the parties. 
+On the sending party:
+```
+transmat --send <FILENAME>
+```
+This will prepare the sender and output the command to run on the receiving party, eg:
+```
+transmat --receive --password Space-Time-Continuum
+```
+
+The sending party will wait for the receiving party to connect to the relay server, and then it will start the transfer. \
+With the exception of some service messages, all the data is encrypted. \
+Encryption and decryption is done only client-side, and the relay server has no useful knowledge of the data it relays. \
+Fernet module (AES128-CBC + HMAC-SHA256) is used to encrypt and authenticate the data.
+
+See also ```transmat --help```

transmat/transmat.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import asyncio, websockets
-import sys, os, base64, argparse, json, pickle
+import sys, os, base64, argparse, json, pickle, math
 from cryptography.fernet import Fernet
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
@@ -9,7 +9,7 @@ from progress.bar import Bar
 import diceware
 
 
-VERSION_NUMBER="0.1.3"
+VERSION_NUMBER="0.1.8"
 SCRIPT_BASENAME = os.path.basename(sys.argv[0])
 
 async def read_chunks(filename, chunk_size = 1024):
@@ -95,7 +95,7 @@ async def send_encrypted_msg(ws, k, data):
 
 
 async def main():
-    WS_RELAY_SERVER = "wss://transmat.exocortex.ru"
+    WS_RELAY_SERVER = "wss://transmat.exocortex.ru/ws"
     parser = argparse.ArgumentParser()
     arg_group = parser.add_mutually_exclusive_group(required=True)
     arg_group.add_argument('--receive', '--recv', action='store_true', help='Receive a file from the remote party (mutually exclusive with --send and --relay)')
@@ -137,7 +137,7 @@ async def main():
         file_size = os.path.getsize(file_path)
         chunk_id = 0
         chunk_size = 1024 * 512
-        number_of_chunks = round(file_size / chunk_size)
+        number_of_chunks = math.ceil(file_size / chunk_size)
         WS_RELAY_SERVER = WS_RELAY_SERVER.replace('http', 'ws', 1)
         async with websockets.connect(WS_RELAY_SERVER) as ws:
             msgtype = "announce"
@@ -147,12 +147,11 @@ async def main():
                 message = json.loads(message)
                 if message["msgtype"] == "announce" and message["peer_group_id"] == peer_group_id:
                     break
-            bar = Bar('Transferring', max=number_of_chunks,  suffix='%(percent).1f%% complete - %(eta_td)s remaining')
+            bar = Bar(filename, max=number_of_chunks, check_tty=False, suffix='%(percent).1f%% complete - %(eta_td)s remaining')
             msgtype = "data"
             async for chunk in read_chunks(file_path, chunk_size):
                 msg = (msgtype, peer_group_id, role, filename, chunk_size, chunk_id, number_of_chunks, chunk)
                 await send_encrypted_msg(ws, k, msg)
-                await asyncio.sleep(0.05)
                 chunk_id += 1
                 if chunk_id > number_of_chunks:
                     break
@@ -168,32 +167,34 @@ async def main():
             await send_encrypted_msg(ws, k, (msgtype, peer_group_id, role, "", "", "", "", ""))
             bar = None
             f = None
-            i = 1
             while True:
                 message = await ws.recv()
                 message = json.loads(message)
                 payload = message["payload"]
                 msg = pickle.loads(decrypt_chunk(k, payload))
+                chunk_id = msg["chunk_id"]
+                number_of_chunks = msg["number_of_chunks"]
+                filename = msg["filename"]
                 if bar is None:
-                    filename = msg["filename"]
-                    number_of_chunks = msg["number_of_chunks"]
-                    bar = Bar('Receiving', max=number_of_chunks,  suffix='%(percent).1f%% complete - %(eta_td)s remaining')
+                    bar = Bar(filename, max=number_of_chunks,  check_tty=False, suffix='%(percent).1f%% complete - %(eta_td)s remaining')
                 if f is None:
-                    f = open(msg["filename"], "wb")
+                    f = open(filename, "wb")
                     f.write(msg["chunk"])
                 else:
                     f.write(msg["chunk"])
                 msgtype = "proceed"
                 await send_encrypted_msg(ws, k, (msgtype, peer_group_id, role, "", "", "", "", "")) # request the next chunk from sender
-                i += 1
+                bar.suffix = "%(percent).1f%% complete - %(eta_td)s remaining"
                 bar.next()
-                if i > msg["number_of_chunks"] + 1:
-                    bar.index = 100
+
+                if chunk_id >= number_of_chunks-1:
+                    # This was the last chunk, exit
+                    f.close()
                     bar.suffix = '%(percent).1f%% complete'
                     bar.update()
-                    f.close()
-                    print("")
+                    bar.finish()
                     break
+ 
                 
 
 asyncio.run(main())

transphase/docs/Message.md

@@ -0,0 +1,17 @@
+# Transmat message format
+## ver. 0.1
+
+### Message
+* msgtype (announce, data, confirmation)
+* per_group_id
+* role (send, receive)
+* payload (encrypted payload containing file chunks and service metadata)
+* *TODO - Add protocol version number*
+
+### Payload
+* filename (current filename without path)
+* chunk_size (size of each chunk)
+* chunk_id (number of current chunk)
+* numbr of chunks (total number of chunks in file)
+* chunk (current data)
+* *TODO - Add a hash of the chunk data to use in confirmation messages*

transphase/docs/Protocol.md

@@ -0,0 +1,22 @@
+# Transmat protocol description
+## ver. 0.1
+
+### **Dramatis Personae:**
+*Relay* - Transphase relay Relay
+*Sender* - sending party
+*Receiver* - receiving party
+
+* *Sender* creates a low-entropy password, derives a key and a `peer_group_id` vaule and sends the message containing the `peer_group_id` to the *Relay*
+* *Relay*  maintains a list of all its peers grouped by their `peer_group_id` value. 
+    * If the current message has a new unique `peer_group_id`, then a new peer group is created.
+    * If the current message has a known `peer_group_id`, then a new peer group is created from it and the party that sent the message (normally the *Sender*) is added to that group.
+* *Sender* passes the low-entropy password to the *Receiver* over a different channel (e.g. phone or SSH).
+* *Receiver* derives a key and a `peer_group_id` value and sends the message containing the `peer_group_id` to the *Relay*.
+    * This message contains 0 as `chunk_number` so that the *Sender* knows that the transfer should start from the very beginning.
+* *Relay* forwards this message to all other peers in the corresponding peer group (that is to the *Sender*).
+* *Sender* receives this message; sends the encrypted chunk and metadata; and then waits for the *Receiver* to send a confirmation that it has successfully received the  message. 
+* *Relay* forwards this message to all other peers in the corresponding peer group (that is to the *Receiver*).
+* *Receiver* decrypts the payload and writes it to the target file. (TODO: Check chunk hash) Then it sends a confirmation to the *Relay*. The confirmation contains the incremented number as the `chunk_id`
+    * If this was the last chunk in the transmission, *Receiver* closes the Websocket connection to the *Relay*
+* *Sender* receives the confirmation and sends the next encrypted chunk and metadata.
+    * If *Sender* has no more chunks to send, *Sender* closes the Websocket connection to the *Relay*

transphase/requirements.txt

@@ -1,6 +1,2 @@
-asyncio==3.4.3
-cffi==1.15.1
 cryptography==40.0.2
-pycparser==2.21
-pyjson==1.3.0
-websockets==11.0.2
+quart

transphase/transphase.py

@@ -1,58 +1,69 @@
-#!/usr/bin/env python
-import asyncio
-import websockets
-import logging 
 import json
-
-
-logging.basicConfig(
-    format="%(asctime)s %(message)s",
-    level=logging.INFO,
-)
+import asyncio
+import sys, time
+from quart import Quart, websocket
 
 peer_list = {}
+TRANSFERRED_DATA = 0
 
-class LoggerAdapter(logging.LoggerAdapter):
-    """Add connection ID and client IP address to websockets logs."""
-    def process(self, msg, kwargs):
-        try:
-            websocket = kwargs["extra"]["websocket"]
-        except KeyError:
-            return msg, kwargs
-        rip = websocket.remote_address[0]
-        try: 
-            xff = websocket.request_headers.get("X-Forwarded-For")
-        except:
-            xff = "None"
-        return f"{websocket.id} {rip} {xff}", kwargs
+app = Quart(__name__)
 
-async def handler(websocket):
+
+def convert_bytes(byte_value):
+    if byte_value < 1024:  # Less than 1 kilobyte
+        return str(byte_value) +  " B"
+    elif byte_value < 1024**2:  # Less than 1 megabyte
+        kb_value = round(byte_value / 1024)
+        return str(kb_value) + " KB"
+    elif byte_value < 1024**3:  # Less than 1 gigabyte
+        mb_value = round(byte_value / (1024**2),2)
+        return str(mb_value) + " MB"
+    elif byte_value < 1024**4:  # Less than 1 terabyte
+        gb_value = round(byte_value / (1024**3),2)
+        return str(gb_value) + " GB"
+    elif byte_value < 1024**5:  # Less than 1 petabyte
+        tb_value = round(byte_value / (1024**4),2)
+        return str(tb_value) + " TB"
+    else:  # More than or equal to 1 petabyte
+        tb_value = round(byte_value / (1024**5),2)
+        return str(tb_value) + " PB"
+
+@app.route("/")
+async def retmain():
+    return f"Ready to relay\n"
+
+@app.websocket("/ws")
+async def handle_websockets():
+    global TRANSFERRED_DATA
     peer_group_id = None
-    try:
-        while not websocket.closed:
-            message = await websocket.recv()
+    while True:
+        try:
+            message = await websocket.receive()
             msg = json.loads(message)
             if "peer_group_id" in msg:
                 peer_group_id = msg["peer_group_id"]
                 if peer_group_id not in peer_list:
-                    peer_list[peer_group_id] = set() # Create new set for all peers in peer_group_id
-                peer_list[peer_group_id].add(websocket) # Add peer's socket to peer_group
+                    peer_list[peer_group_id] = set()
+                peer_list[peer_group_id].add(websocket._get_current_object())
             for peer in peer_list[peer_group_id]:
-                if peer != websocket:
+                if peer != websocket._get_current_object():
                     await peer.send(message)
-    except websockets.exceptions.ConnectionClosed as e:
-        pass
-    finally:
-        peer_list[peer_group_id].remove(websocket)
-        if len(peer_list[peer_group_id]) < 1:
-            peer_list.pop(peer_group_id)
+                    TRANSFERRED_DATA = TRANSFERRED_DATA + sys.getsizeof(message)
+        except asyncio.exceptions.CancelledError:
+            peer_list[peer_group_id].remove(websocket._get_current_object())
+            if len(peer_list[peer_group_id]) < 1:
+                peer_list.pop(peer_group_id)
 
-async def main():
-    async with websockets.serve(
-        handler, "", 8001, ping_timeout=30,
-        logger=LoggerAdapter(logging.getLogger("websockets.server"), None),
-        ):
-        await asyncio.Future()  # run forever
+@app.route("/stats")
+async def return_stats():
+    process_start_time = time.process_time()
+    current_time = time.time()
+    uptime = current_time - process_start_time
+    uptime_str = time.strftime("%H:%M:%S", time.gmtime(uptime))
+    peers_connected = 0
+    for p in peer_list.values():
+        peers_connected += len(p)
+    resp = f"<p><b>Uptime: </b>{uptime_str}</p><p><b>Peers: </b>{peers_connected}</p><p><b>Transferred: </b>{convert_bytes(TRANSFERRED_DATA)}</p>"
+    return resp
 
-if __name__ == "__main__":
-    asyncio.run(main())
+app.run()