improve documentation

doc/index.rst

@@ -8,6 +8,10 @@ Welcome to blockchain's documentation!
 
 This project is a completely new blockchain-based coin, with P2P networking, a consensus mechanism and a wallet interface. The goal of the project is to provide a framework that is easy to modify for people who want to develop proof-of-concepts for blockchain-based technology.
 
+DO NOT USE THIS AS A REAL CURRENCY TO SEND, RETRIEVE, OR STORE ACTUAL MONEY! While we do not currently know of any way to do so, there are almost certainly
+bugs in this implementation that would allow anyone to create money out of the blue or take yours away from you.
+
+
 Executables
 ***********
 
@@ -20,6 +24,37 @@ Executables
     * - wallet
       - .. automodule:: wallet
 
+To start a minimal network of two peers that do not mine, you can do this on different machines::
+
+    ./miner.py --listen-port 1234
+    ./miner.py --bootstrap-peer a.b.c.d:1234
+
+To actually start mining, you'll need to use the wallet and generate a new address that should receive the mining rewards::
+
+    ./wallet.py --wallet mining.wallet create-address mining-address.pem
+
+Afterwards, you can copy the file `mining-address.pem` to the second machine and restart the miner like this::
+
+    ./miner.py --bootstrap-peer a.b.c.d:1234 --mining-pubkey mining-address.pem
+
+This miner will now mine new blocks for the block chain and send them to the miner application on
+the other machine. Once some blocks have been mined, you can check how much money you have made like this::
+
+    ./wallet.py --wallet mining.wallet show-balance
+
+Once you have earned money mining, you can send some of it to someone else. To send them 42 coins (we already know how the other person can generate an address), you can do this::
+
+    ./wallet.py --wallet mining.wallet transfer other_person_address.pem 42
+
+Both the miner and the wallet have many more options than just these, which can be found using the `--help` switch of the programs. Especially useful might also be the `--rpc-port` option of the miner, which needs to be set to different values when one wants to start more than one instance on the same computer::
+
+    ./miner.py --rpc-port 2345 --listen-port 1234
+    ./miner.py --rpc-port 3456 --bootstrap-peer 127.0.0.1:1234
+    ./wallet.py --miner-port 2345 --wallet mining.wallet show-balance
+
+
+
+
 
 Source Code Documentation
 *************************

src/block.py

@@ -14,7 +14,10 @@ __all__ = ['Block', 'GENESIS_BLOCK', 'GENESIS_BLOCK_HASH']
 
 class Block:
     """
-    A block.
+    A block: a container for all the data associated with a block.
+
+    To figure out whether the block is valid on top of a block chain, there are a few `verify`
+    methods. Without calling these, you must assume the block was crafted maliciously.
 
     :ivar hash: The hash value of this block.
     :vartype hash: bytes

src/blockchain.py

@@ -15,7 +15,10 @@ REWARD_HALF_LIFE = 10000
 
 class Blockchain:
     """
-    A block chain: a ordered list of valid blocks.
+    A block chain: a ordered, immutable list of valid blocks. The only ways to create a blockchain
+    instance are the constructor, which will create a block chain containing only the genesis block,
+    and the `try_append` method which creates a new block chain only if the given block is valid on
+    top of `self`.
 
     :ivar blocks: The blocks in this chain, oldest first.
     :vartype blocks: List[Block]

src/chainbuilder.py

@@ -1,6 +1,30 @@
 """
-The chain builder maintains the current longest confirmed (primary) block chain as well as one
-candidate for an even longer chain that it attempts to download and verify.
+The chain builder maintains the current longest confirmed (primary) block chain as well as
+only partially downloaded longer chains that might become the new primary block chain once
+completed and verified. Also maintains a list of (unconfirmed) transactions that are valid
+but not yet part of the primary block chain, or valid once another unconfirmed transaction
+becomes valid.
+
+Received blocks that cannot be shown to be invalid on *any* block chain are stored in a block
+cache (from which for the moment they are never evicted), so that they do not need to be requested
+from other peers over and over again.
+
+
+For the process of building new primary block chains, block requests are used. These are maintained
+in a dict indexed by the hash of the next block that is required for the block request to make
+progress. Each block request stores a list of partial block chains that all depend on the same
+next block, and the time of the last download request, so that these requests can be retried and
+at some point aborted when no progress is made.
+
+While not strictly necessary, the block requests are also used when the block can be found in the
+block cache. In that case they are immediately fulfilled until the block chains can be built or a
+block is missing in the cache, which then will be requested from the peers.
+
+Partial chains are completed once their next block is the head of a so called `checkpoint`. These
+checkpoints are snapshots of the primary block chain at various points in its history. For a chain
+of length `N`, the number of checkpoints is always kept between `2*log_2(N)` and `log_2(N)`, with
+most checkpoints being relatively recent. There also is always one checkpoint with only the genesis
+block.
 """
 
 import threading
@@ -67,8 +91,11 @@ class BlockRequest:
 
 class ChainBuilder:
     """
-    Maintains the current longest confirmed (primary) block chain as well as one candidate for an
-    even longer chain that it attempts to download and verify.
+    The chain builder maintains the current longest confirmed (primary) block chain as well as
+    only partially downloaded longer chains that might become the new primary block chain once
+    completed and verified. Also maintains a list of (unconfirmed) transactions that are valid
+    but not yet part of the primary block chain, or valid once another unconfirmed transaction
+    becomes valid.
 
     :ivar primary_block_chain: The longest fully validated block chain we know of.
     :vartype primary_block_chain: Blockchain

src/crypto.py

@@ -17,7 +17,8 @@ def get_hasher():
     return SHA512.new()
 
 
-MAX_HASH = (1 << 512) - 1 # the largest possible hash value
+MAX_HASH = (1 << 512) - 1
+""" The largest possible hash value, when interpreted as an unsigned int. """
 
 
 class Signing:

src/mining.py

@@ -79,7 +79,19 @@ def wait_for_result(pipes: List[int], cls: type):
 
 class Miner:
     """
-    Management of a background thread that mines for new blocks.
+    Management of a background process that mines for new blocks.
+
+    The miner process is forked for each new proof of work that needs to be performed. The
+    completed block is sent back JSON-serialized through a pipe that is opened for that purpose.
+    When that pipe is closed by the parent process prematurely, the proof of work process knows it
+    is no longer needed and exits.
+
+    To start the mining process, `start_mining` needs to be called once. After that, the mining
+    will happen automatically, with the mined block switching every time the chainbuilder finds a
+    new primary block chain.
+
+    To stop the mining process, there is the `shutdown` method. Once stopped, mining cannot be
+    resumed (except by creating a new `Miner`).
 
     :ivar proto: The protocol where newly mined blocks will be sent to.
     :vartype proto: Protocol

src/persistence.py

@@ -16,8 +16,13 @@ class Persistence:
     """
     Functionality for storing and retrieving the miner state on disk.
 
-    :param path: The path to the storage location. """#TODO
-    """:param chainbuilder: The chainbuilder to persist.
+    Right now, this class stores the connected peers, the blocks in the primary block chain and the
+    unconfirmed transactions in a gzip-compressed file in JSON format. These are dumped whenever
+    the primary block chain changes or a new unconfirmed transaction is received, but at most every
+    `PERSISTENCE_MIN_INTERVAL` time steps.
+
+    :param path: The path to the storage location.
+    :param chainbuilder: The chainbuilder to persist.
     """
     def __init__(self, path: str, chainbuilder: 'ChainBuilder'):
         self.chainbuilder = chainbuilder

src/protocol.py

@@ -1,4 +1,20 @@
-""" Implementation of the P2P protocol. """
+"""
+Implementation of the P2P protocol.
+
+The protocol is text-based and works over TCP. Once a TCP connection is established, there is no
+difference between server and client. Both sides start by sending a fixed `HELLO_MSG` to make sure
+they speak the same protocol. After that, they can send any number of messages.
+
+Messages start with a length (ending with a new-line), followed by the JSON-encoded contents of
+the message of that length. On the top level, the sent JSON values are always dicts, with a
+'msg_type' key indicating the kind of message and a 'msg_param' key containing a type-specific
+payload.
+
+To make sure that the peer acting as a TCP server in a connection knows how to reach the TCP client,
+there is a 'myport' message containing the TCP port where a peer listens for incoming connections.
+
+For other message types, you can look at the `received_*` methods of `Protocol`.
+"""
 
 import json
 import socket
@@ -20,7 +36,11 @@ MAX_PEERS = 10
 """ The maximum number of peers that we connect to."""
 
 HELLO_MSG = b"bl0ckch41n" + hexlify(GENESIS_BLOCK_HASH)[:30] + b"\n"
-""" The hello message two peers use to make sure they are speaking the same protocol. """
+"""
+The hello message two peers use to make sure they are speaking the same protocol. Contains the
+genesis block hash, so that communication of incompatible forks of the program is less likely to
+succeed.
+"""
 
 SOCKET_TIMEOUT = 30
 """ The socket timeout for P2P connections. """
@@ -28,6 +48,7 @@ SOCKET_TIMEOUT = 30
 class PeerConnection:
     """
     Handles the low-level socket connection to one other peer.
+
     :ivar peer_addr: The self-reported address one can use to connect to this peer.
     :ivar param: The self-reported address one can use to connect to this peer.
     :ivar _sock_addr: The address our socket is or will be connected to.
@@ -176,9 +197,7 @@ class PeerConnection:
 
 
 class SocketServer(socketserver.TCPServer):
-    """
-    A TCP socketserver that calls does not close the connections on its own.
-    """
+    """ A TCP socketserver that does not close connections when the handler returns. """
 
     allow_reuse_address = True
     """ Make sure the server can be restarted without delays. """
@@ -211,6 +230,9 @@ class Protocol:
     """
 
     _dummy_peer = namedtuple("DummyPeerConnection", ["peer_addr"])("self")
+    """
+    A dummy peer for messages that are injected by this program, not received from a remote peer.
+    """
 
     def __init__(self, bootstrap_peers: 'List[tuple]',
                  primary_block: 'Block', listen_port: int=0, listen_addr: str=""):

src/rpc_server.py

@@ -10,7 +10,7 @@ from .crypto import Signing
 from .transaction import TransactionInput
 
 def rpc_server(port: int, chainbuilder: ChainBuilder, persist: Persistence):
-    """ Runs the RPC server. """
+    """ Runs the RPC server (forever). """
 
     app = flask.Flask(__name__)
 

src/transaction.py

@@ -45,7 +45,9 @@ class TransactionInput(namedtuple("TransactionInput", ["transaction_hash", "outp
 
 class Transaction:
     """
-    A transaction.
+    A transaction as it was received from the network or a block. To check that this transaction is
+    valid on top of a block chain and in combination with a set of different transactions (from the
+    same block), you can call the `verify` method.
 
     :ivar inputs: The inputs of this transaction. Empty in the case of block reward transactions.
     :vartype inputs: List[TransactionInput]