Add initial state machine rewrite code

This is the first drop of the new state machine code.

The rewrite has the following objectives:
- Remove duplication of state code between client and server
- Remove duplication of state code between TLS and DTLS
- Simplify transitions and bring the logic together in a single location
  so that it is easier to validate
- Remove duplication of code between each of the message handling functions
- Receive a message first and then work out whether that is a valid
  transition - not the other way around (the other way causes lots of issues
  where we are expecting one type of message next but actually get something
  else)
- Separate message flow state from handshake state (in order to better
  understand each)
  - message flow state = when to flush buffers; handling restarts in the
    event of NBIO events; handling the common flow of steps for reading a
    message and the common flow of steps for writing a message etc
  - handshake state = what handshake message are we working on now
- Control complexity: only the state machine can change state: keep all
  the state changes local to a file

This builds on previous state machine related work:
- Surface CCS processing in the state machine
- Version negotiation rewrite
Reviewed-by: NTim Hudson <tjh@openssl.org>
Reviewed-by: NRichard Levitte <levitte@openssl.org>

include/openssl/ssl.h

@@ -1928,6 +1928,7 @@ void ERR_load_SSL_strings(void);
 # define SSL_F_DTLS1_SEND_SERVER_HELLO                    266
 # define SSL_F_DTLS1_SEND_SERVER_KEY_EXCHANGE             267
 # define SSL_F_DTLS1_WRITE_APP_DATA_BYTES                 268
+# define SSL_F_READ_STATE_MACHINE                         352
 # define SSL_F_SSL3_ACCEPT                                128
 # define SSL_F_SSL3_ADD_CERT_TO_BUF                       296
 # define SSL_F_SSL3_CALLBACK_CTRL                         233
@@ -2085,6 +2086,7 @@ void ERR_load_SSL_strings(void);
 # define SSL_F_SSL_USE_RSAPRIVATEKEY_FILE                 206
 # define SSL_F_SSL_VERIFY_CERT_CHAIN                      207
 # define SSL_F_SSL_WRITE                                  208
+# define SSL_F_STATE_MACHINE                              353
 # define SSL_F_TLS12_CHECK_PEER_SIGALG                    333
 # define SSL_F_TLS1_CERT_VERIFY_MAC                       286
 # define SSL_F_TLS1_CHANGE_CIPHER_STATE                   209

ssl/Makefile

@@ -26,7 +26,8 @@ LIBSRC=	\
 	ssl_ciph.c ssl_stat.c ssl_rsa.c \
 	ssl_asn1.c ssl_txt.c ssl_algs.c ssl_conf.c \
 	bio_ssl.c ssl_err.c t1_reneg.c tls_srp.c t1_trce.c ssl_utst.c \
-	record/ssl3_buffer.c record/ssl3_record.c record/dtls1_bitmap.c
+	record/ssl3_buffer.c record/ssl3_record.c record/dtls1_bitmap.c \
+	statem.c
 LIBOBJ= \
 	s3_srvr.o  s3_clnt.o  s3_lib.o  s3_enc.o record/rec_layer_s3.o \
 	s3_both.o s3_cbc.o s3_msg.o \
@@ -37,7 +38,8 @@ LIBOBJ= \
 	ssl_ciph.o ssl_stat.o ssl_rsa.o \
 	ssl_asn1.o ssl_txt.o ssl_algs.o ssl_conf.o \
 	bio_ssl.o ssl_err.o t1_reneg.o tls_srp.o t1_trce.o ssl_utst.o \
-	record/ssl3_buffer.o record/ssl3_record.o record/dtls1_bitmap.o
+	record/ssl3_buffer.o record/ssl3_record.o record/dtls1_bitmap.o \
+	statem.o
 
 SRC= $(LIBSRC)
 
@@ -781,6 +783,26 @@ ssl_utst.o: ../include/openssl/stack.h ../include/openssl/symhacks.h
 ssl_utst.o: ../include/openssl/tls1.h ../include/openssl/x509.h
 ssl_utst.o: ../include/openssl/x509_vfy.h packet_locl.h record/record.h
 ssl_utst.o: ssl_locl.h ssl_utst.c
+statem.o: ../e_os.h ../include/openssl/asn1.h ../include/openssl/bio.h
+statem.o: ../include/openssl/bn.h ../include/openssl/buffer.h
+statem.o: ../include/openssl/comp.h ../include/openssl/crypto.h
+statem.o: ../include/openssl/dsa.h ../include/openssl/dtls1.h
+statem.o: ../include/openssl/e_os2.h ../include/openssl/ec.h
+statem.o: ../include/openssl/ecdh.h ../include/openssl/ecdsa.h
+statem.o: ../include/openssl/err.h ../include/openssl/evp.h
+statem.o: ../include/openssl/hmac.h ../include/openssl/lhash.h
+statem.o: ../include/openssl/obj_mac.h ../include/openssl/objects.h
+statem.o: ../include/openssl/opensslconf.h ../include/openssl/opensslv.h
+statem.o: ../include/openssl/ossl_typ.h ../include/openssl/pem.h
+statem.o: ../include/openssl/pem2.h ../include/openssl/pkcs7.h
+statem.o: ../include/openssl/pqueue.h ../include/openssl/rand.h
+statem.o: ../include/openssl/rsa.h ../include/openssl/safestack.h
+statem.o: ../include/openssl/sha.h ../include/openssl/srtp.h
+statem.o: ../include/openssl/ssl.h ../include/openssl/ssl2.h
+statem.o: ../include/openssl/ssl3.h ../include/openssl/stack.h
+statem.o: ../include/openssl/symhacks.h ../include/openssl/tls1.h
+statem.o: ../include/openssl/x509.h ../include/openssl/x509_vfy.h packet_locl.h
+statem.o: record/record.h ssl_locl.h statem.c
 t1_clnt.o: ../e_os.h ../include/openssl/asn1.h ../include/openssl/bio.h
 t1_clnt.o: ../include/openssl/bn.h ../include/openssl/buffer.h
 t1_clnt.o: ../include/openssl/comp.h ../include/openssl/crypto.h

ssl/s3_both.c

@@ -478,6 +478,7 @@ long ssl3_get_message(SSL *s, int st1, int stn, int mt, long max, int *ok)
     return n;
  f_err:
     ssl3_send_alert(s, SSL3_AL_FATAL, al);
+    statem_set_error(s);
     *ok = 0;
     return 0;
 }

ssl/s3_lib.c

@@ -5132,6 +5132,7 @@ int ssl3_renegotiate_check(SSL *s)
              */
             /* SSL_ST_ACCEPT */
             s->state = SSL_ST_RENEGOTIATE;
+            statem_set_renegotiate(s);
             s->s3->renegotiate = 0;
             s->s3->num_renegotiations++;
             s->s3->total_renegotiations++;

ssl/ssl_err.c

@@ -112,6 +112,7 @@ static ERR_STRING_DATA SSL_str_functs[] = {
     {ERR_FUNC(SSL_F_DTLS1_SEND_SERVER_KEY_EXCHANGE),
      "dtls1_send_server_key_exchange"},
     {ERR_FUNC(SSL_F_DTLS1_WRITE_APP_DATA_BYTES), "dtls1_write_app_data_bytes"},
+    {ERR_FUNC(SSL_F_READ_STATE_MACHINE), "READ_STATE_MACHINE"},
     {ERR_FUNC(SSL_F_SSL3_ACCEPT), "ssl3_accept"},
     {ERR_FUNC(SSL_F_SSL3_ADD_CERT_TO_BUF), "SSL3_ADD_CERT_TO_BUF"},
     {ERR_FUNC(SSL_F_SSL3_CALLBACK_CTRL), "ssl3_callback_ctrl"},
@@ -313,6 +314,7 @@ static ERR_STRING_DATA SSL_str_functs[] = {
     {ERR_FUNC(SSL_F_SSL_USE_RSAPRIVATEKEY_FILE), "SSL_use_RSAPrivateKey_file"},
     {ERR_FUNC(SSL_F_SSL_VERIFY_CERT_CHAIN), "ssl_verify_cert_chain"},
     {ERR_FUNC(SSL_F_SSL_WRITE), "SSL_write"},
+    {ERR_FUNC(SSL_F_STATE_MACHINE), "STATE_MACHINE"},
     {ERR_FUNC(SSL_F_TLS12_CHECK_PEER_SIGALG), "tls12_check_peer_sigalg"},
     {ERR_FUNC(SSL_F_TLS1_CERT_VERIFY_MAC), "tls1_cert_verify_mac"},
     {ERR_FUNC(SSL_F_TLS1_CHANGE_CIPHER_STATE), "tls1_change_cipher_state"},

ssl/ssl_lib.c

@@ -218,6 +218,7 @@ int SSL_clear(SSL *s)
     s->type = 0;
 
     s->state = SSL_ST_BEFORE | ((s->server) ? SSL_ST_ACCEPT : SSL_ST_CONNECT);
+    statem_clear(s);
 
     s->version = s->method->version;
     s->client_version = s->version;

ssl/ssl_locl.h

@@ -717,6 +717,137 @@ struct ssl_comp_st {
 DECLARE_STACK_OF(SSL_COMP)
 DECLARE_LHASH_OF(SSL_SESSION);
 
+/*
+ * The valid handshake states (one for each type message sent and one for each
+ * type of message received). There are also two "special" states:
+ * TLS = TLS or DTLS state
+ * DTLS = DTLS specific state
+ * CR/SR = Client Read/Server Read
+ * CW/SW = Client Write/Server Write
+ *
+ * The "special" states are:
+ * TLS_ST_BEFORE = No handshake has been initiated yet
+ * TLS_ST_OK = A handshake has been successfully completed
+ */
+enum HANDSHAKE_STATE {
+    TLS_ST_BEFORE,
+    TLS_ST_OK,
+    DTLS_ST_CR_HELLO_VERIFY_REQUEST,
+    TLS_ST_CR_SRVR_HELLO,
+    TLS_ST_CR_CERT,
+    TLS_ST_CR_CERT_STATUS,
+    TLS_ST_CR_KEY_EXCH,
+    TLS_ST_CR_CERT_REQ,
+    TLS_ST_CR_SRVR_DONE,
+    TLS_ST_CR_SESSION_TICKET,
+    TLS_ST_CR_CHANGE,
+    TLS_ST_CR_FINISHED,
+    TLS_ST_CW_CLNT_HELLO,
+    TLS_ST_CW_CERT,
+    TLS_ST_CW_KEY_EXCH,
+    TLS_ST_CW_CERT_VRFY,
+    TLS_ST_CW_CHANGE,
+    TLS_ST_CW_NEXT_PROTO,
+    TLS_ST_CW_FINISHED,
+    TLS_ST_SW_HELLO_REQ,
+    TLS_ST_SR_CLNT_HELLO,
+    DTLS_ST_SW_HELLO_VERIFY_REQUEST,
+    TLS_ST_SW_SRVR_HELLO,
+    TLS_ST_SW_CERT,
+    TLS_ST_SW_KEY_EXCH,
+    TLS_ST_SW_CERT_REQ,
+    TLS_ST_SW_SRVR_DONE,
+    TLS_ST_SR_CERT,
+    TLS_ST_SR_KEY_EXCH,
+    TLS_ST_SR_CERT_VRFY,
+    TLS_ST_SR_NEXT_PROTO,
+    TLS_ST_SR_CHANGE,
+    TLS_ST_SR_FINISHED,
+    TLS_ST_SW_SESSION_TICKET,
+    TLS_ST_SW_CERT_STATUS,
+    TLS_ST_SW_CHANGE,
+    TLS_ST_SW_FINISHED
+};
+
+/*
+ * Valid return codes used for functions performing work prior to or after
+ * sending or receiving a message
+ */
+enum WORK_STATE {
+    /* Something went wrong */
+    WORK_ERROR,
+    /* We're done working and there shouldn't be anything else to do after */
+    WORK_FINISHED_STOP,
+    /* We're done working move onto the next thing */
+    WORK_FINISHED_CONTINUE,
+    /* We're working on phase A */
+    WORK_MORE_A,
+    /* We're working on phase B */
+    WORK_MORE_B
+};
+
+/* Write transition return codes */
+enum WRITE_TRAN {
+    /* Something went wrong */
+    WRITE_TRAN_ERROR,
+    /* A transition was successfully completed and we should continue */
+    WRITE_TRAN_CONTINUE,
+    /* There is no more write work to be done */
+    WRITE_TRAN_FINISHED
+};
+
+/* Message processing return codes */
+enum MSG_PROCESS_RETURN {
+    MSG_PROCESS_ERROR,
+    MSG_PROCESS_FINISHED_READING,
+    MSG_PROCESS_CONTINUE_PROCESSING,
+    MSG_PROCESS_CONTINUE_READING
+};
+
+/* Message flow states */
+enum MSG_FLOW_STATE {
+    /* No handshake in progress */
+    MSG_FLOW_UNINITED,
+    /* A permanent error with this connection */
+    MSG_FLOW_ERROR,
+    /* We are about to renegotiate */
+    MSG_FLOW_RENEGOTIATE,
+    /* We are reading messages */
+    MSG_FLOW_READING,
+    /* We are writing messages */
+    MSG_FLOW_WRITING,
+    /* Handshake has finished */
+    MSG_FLOW_FINISHED
+};
+
+/* Read states */
+enum READ_STATE {
+    READ_STATE_HEADER,
+    READ_STATE_BODY,
+    READ_STATE_POST_PROCESS
+};
+
+/* Write states */
+enum WRITE_STATE {
+    WRITE_STATE_TRANSITION,
+    WRITE_STATE_PRE_WORK,
+    WRITE_STATE_SEND,
+    WRITE_STATE_POST_WORK
+};
+
+struct statem_st {
+    enum MSG_FLOW_STATE state;
+    enum WRITE_STATE write_state;
+    enum WORK_STATE write_state_work;
+    enum READ_STATE read_state;
+    enum WORK_STATE read_state_work;
+    enum HANDSHAKE_STATE hand_state;
+    int read_state_first_init;
+    int use_timer;
+};
+typedef struct statem_st STATEM;
+
+
 struct ssl_ctx_st {
     const SSL_METHOD *method;
     STACK_OF(SSL_CIPHER) *cipher_list;
@@ -1012,6 +1143,8 @@ struct ssl_st {
     int shutdown;
     /* where we are */
     int state;
+    STATEM statem;
+
     BUF_MEM *init_buf;          /* buffer used during init */
     void *init_msg;             /* pointer to handshake message body, set by
                                  * ssl3_get_message() */
@@ -1951,6 +2084,10 @@ __owur int ssl3_new(SSL *s);
 void ssl3_free(SSL *s);
 __owur int ssl3_accept(SSL *s);
 __owur int ssl3_connect(SSL *s);
+void statem_clear(SSL *s);
+void statem_set_renegotiate(SSL *s);
+void statem_set_error(SSL *s);
+__owur int statem_client_app_data_allowed(SSL *s);
 __owur int ssl3_read(SSL *s, void *buf, int len);
 __owur int ssl3_peek(SSL *s, void *buf, int len);
 __owur int ssl3_write(SSL *s, const void *buf, int len);