Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687
  1. #include <string.h>
  2. #include <stdlib.h>
  3. #include <stdio.h>
  4. #include <assert.h>
  5. #include <errno.h>
  6. #include <byteswap.h>
  7. #include <ipxe/timer.h>
  8. #include <ipxe/iobuf.h>
  9. #include <ipxe/malloc.h>
  10. #include <ipxe/init.h>
  11. #include <ipxe/retry.h>
  12. #include <ipxe/refcnt.h>
  13. #include <ipxe/pending.h>
  14. #include <ipxe/xfer.h>
  15. #include <ipxe/open.h>
  16. #include <ipxe/uri.h>
  17. #include <ipxe/netdevice.h>
  18. #include <ipxe/profile.h>
  19. #include <ipxe/process.h>
  20. #include <ipxe/tcpip.h>
  21. #include <ipxe/tcp.h>
  22. /** @file
  23. *
  24. * TCP protocol
  25. *
  26. */
  27. FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  28. /** A TCP connection */
  29. struct tcp_connection {
  30. /** Reference counter */
  31. struct refcnt refcnt;
  32. /** List of TCP connections */
  33. struct list_head list;
  34. /** Flags */
  35. unsigned int flags;
  36. /** Data transfer interface */
  37. struct interface xfer;
  38. /** Remote socket address */
  39. struct sockaddr_tcpip peer;
  40. /** Local port */
  41. unsigned int local_port;
  42. /** Maximum segment size */
  43. size_t mss;
  44. /** Current TCP state */
  45. unsigned int tcp_state;
  46. /** Previous TCP state
  47. *
  48. * Maintained only for debug messages
  49. */
  50. unsigned int prev_tcp_state;
  51. /** Current sequence number
  52. *
  53. * Equivalent to SND.UNA in RFC 793 terminology.
  54. */
  55. uint32_t snd_seq;
  56. /** Unacknowledged sequence count
  57. *
  58. * Equivalent to (SND.NXT-SND.UNA) in RFC 793 terminology.
  59. */
  60. uint32_t snd_sent;
  61. /** Send window
  62. *
  63. * Equivalent to SND.WND in RFC 793 terminology
  64. */
  65. uint32_t snd_win;
  66. /** Current acknowledgement number
  67. *
  68. * Equivalent to RCV.NXT in RFC 793 terminology.
  69. */
  70. uint32_t rcv_ack;
  71. /** Receive window
  72. *
  73. * Equivalent to RCV.WND in RFC 793 terminology.
  74. */
  75. uint32_t rcv_win;
  76. /** Received timestamp value
  77. *
  78. * Updated when a packet is received; copied to ts_recent when
  79. * the window is advanced.
  80. */
  81. uint32_t ts_val;
  82. /** Most recent received timestamp that advanced the window
  83. *
  84. * Equivalent to TS.Recent in RFC 1323 terminology.
  85. */
  86. uint32_t ts_recent;
  87. /** Send window scale
  88. *
  89. * Equivalent to Snd.Wind.Scale in RFC 1323 terminology
  90. */
  91. uint8_t snd_win_scale;
  92. /** Receive window scale
  93. *
  94. * Equivalent to Rcv.Wind.Scale in RFC 1323 terminology
  95. */
  96. uint8_t rcv_win_scale;
  97. /** Selective acknowledgement list (in host-endian order) */
  98. struct tcp_sack_block sack[TCP_SACK_MAX];
  99. /** Transmit queue */
  100. struct list_head tx_queue;
  101. /** Receive queue */
  102. struct list_head rx_queue;
  103. /** Transmission process */
  104. struct process process;
  105. /** Retransmission timer */
  106. struct retry_timer timer;
  107. /** Shutdown (TIME_WAIT) timer */
  108. struct retry_timer wait;
  109. /** Pending operations for SYN and FIN */
  110. struct pending_operation pending_flags;
  111. /** Pending operations for transmit queue */
  112. struct pending_operation pending_data;
  113. };
  114. /** TCP flags */
  115. enum tcp_flags {
  116. /** TCP data transfer interface has been closed */
  117. TCP_XFER_CLOSED = 0x0001,
  118. /** TCP timestamps are enabled */
  119. TCP_TS_ENABLED = 0x0002,
  120. /** TCP acknowledgement is pending */
  121. TCP_ACK_PENDING = 0x0004,
  122. /** TCP selective acknowledgement is enabled */
  123. TCP_SACK_ENABLED = 0x0008,
  124. };
  125. /** TCP internal header
  126. *
  127. * This is the header that replaces the TCP header for packets
  128. * enqueued on the receive queue.
  129. */
  130. struct tcp_rx_queued_header {
  131. /** SEQ value, in host-endian order
  132. *
  133. * This represents the SEQ value at the time the packet is
  134. * enqueued, and so excludes the SYN, if present.
  135. */
  136. uint32_t seq;
  137. /** Next SEQ value, in host-endian order */
  138. uint32_t nxt;
  139. /** Flags
  140. *
  141. * Only FIN is valid within this flags byte; all other flags
  142. * have already been processed by the time the packet is
  143. * enqueued.
  144. */
  145. uint8_t flags;
  146. /** Reserved */
  147. uint8_t reserved[3];
  148. };
  149. /**
  150. * List of registered TCP connections
  151. */
  152. static LIST_HEAD ( tcp_conns );
  153. /** Transmit profiler */
  154. static struct profiler tcp_tx_profiler __profiler = { .name = "tcp.tx" };
  155. /** Receive profiler */
  156. static struct profiler tcp_rx_profiler __profiler = { .name = "tcp.rx" };
  157. /** Data transfer profiler */
  158. static struct profiler tcp_xfer_profiler __profiler = { .name = "tcp.xfer" };
  159. /* Forward declarations */
  160. static struct process_descriptor tcp_process_desc;
  161. static struct interface_descriptor tcp_xfer_desc;
  162. static void tcp_expired ( struct retry_timer *timer, int over );
  163. static void tcp_wait_expired ( struct retry_timer *timer, int over );
  164. static struct tcp_connection * tcp_demux ( unsigned int local_port );
  165. static int tcp_rx_ack ( struct tcp_connection *tcp, uint32_t ack,
  166. uint32_t win );
  167. /**
  168. * Name TCP state
  169. *
  170. * @v state TCP state
  171. * @ret name Name of TCP state
  172. */
  173. static inline __attribute__ (( always_inline )) const char *
  174. tcp_state ( int state ) {
  175. switch ( state ) {
  176. case TCP_CLOSED: return "CLOSED";
  177. case TCP_LISTEN: return "LISTEN";
  178. case TCP_SYN_SENT: return "SYN_SENT";
  179. case TCP_SYN_RCVD: return "SYN_RCVD";
  180. case TCP_ESTABLISHED: return "ESTABLISHED";
  181. case TCP_FIN_WAIT_1: return "FIN_WAIT_1";
  182. case TCP_FIN_WAIT_2: return "FIN_WAIT_2";
  183. case TCP_CLOSING_OR_LAST_ACK: return "CLOSING/LAST_ACK";
  184. case TCP_TIME_WAIT: return "TIME_WAIT";
  185. case TCP_CLOSE_WAIT: return "CLOSE_WAIT";
  186. default: return "INVALID";
  187. }
  188. }
  189. /**
  190. * Dump TCP state transition
  191. *
  192. * @v tcp TCP connection
  193. */
  194. static inline __attribute__ (( always_inline )) void
  195. tcp_dump_state ( struct tcp_connection *tcp ) {
  196. if ( tcp->tcp_state != tcp->prev_tcp_state ) {
  197. DBGC ( tcp, "TCP %p transitioned from %s to %s\n", tcp,
  198. tcp_state ( tcp->prev_tcp_state ),
  199. tcp_state ( tcp->tcp_state ) );
  200. }
  201. tcp->prev_tcp_state = tcp->tcp_state;
  202. }
  203. /**
  204. * Dump TCP flags
  205. *
  206. * @v flags TCP flags
  207. */
  208. static inline __attribute__ (( always_inline )) void
  209. tcp_dump_flags ( struct tcp_connection *tcp, unsigned int flags ) {
  210. if ( flags & TCP_RST )
  211. DBGC2 ( tcp, " RST" );
  212. if ( flags & TCP_SYN )
  213. DBGC2 ( tcp, " SYN" );
  214. if ( flags & TCP_PSH )
  215. DBGC2 ( tcp, " PSH" );
  216. if ( flags & TCP_FIN )
  217. DBGC2 ( tcp, " FIN" );
  218. if ( flags & TCP_ACK )
  219. DBGC2 ( tcp, " ACK" );
  220. }
  221. /***************************************************************************
  222. *
  223. * Open and close
  224. *
  225. ***************************************************************************
  226. */
  227. /**
  228. * Check if local TCP port is available
  229. *
  230. * @v port Local port number
  231. * @ret port Local port number, or negative error
  232. */
  233. static int tcp_port_available ( int port ) {
  234. return ( tcp_demux ( port ) ? -EADDRINUSE : port );
  235. }
  236. /**
  237. * Open a TCP connection
  238. *
  239. * @v xfer Data transfer interface
  240. * @v peer Peer socket address
  241. * @v local Local socket address, or NULL
  242. * @ret rc Return status code
  243. */
  244. static int tcp_open ( struct interface *xfer, struct sockaddr *peer,
  245. struct sockaddr *local ) {
  246. struct sockaddr_tcpip *st_peer = ( struct sockaddr_tcpip * ) peer;
  247. struct sockaddr_tcpip *st_local = ( struct sockaddr_tcpip * ) local;
  248. struct tcp_connection *tcp;
  249. size_t mtu;
  250. int port;
  251. int rc;
  252. /* Allocate and initialise structure */
  253. tcp = zalloc ( sizeof ( *tcp ) );
  254. if ( ! tcp )
  255. return -ENOMEM;
  256. DBGC ( tcp, "TCP %p allocated\n", tcp );
  257. ref_init ( &tcp->refcnt, NULL );
  258. intf_init ( &tcp->xfer, &tcp_xfer_desc, &tcp->refcnt );
  259. process_init_stopped ( &tcp->process, &tcp_process_desc, &tcp->refcnt );
  260. timer_init ( &tcp->timer, tcp_expired, &tcp->refcnt );
  261. timer_init ( &tcp->wait, tcp_wait_expired, &tcp->refcnt );
  262. tcp->prev_tcp_state = TCP_CLOSED;
  263. tcp->tcp_state = TCP_STATE_SENT ( TCP_SYN );
  264. tcp_dump_state ( tcp );
  265. tcp->snd_seq = random();
  266. INIT_LIST_HEAD ( &tcp->tx_queue );
  267. INIT_LIST_HEAD ( &tcp->rx_queue );
  268. memcpy ( &tcp->peer, st_peer, sizeof ( tcp->peer ) );
  269. /* Calculate MSS */
  270. mtu = tcpip_mtu ( &tcp->peer );
  271. if ( ! mtu ) {
  272. DBGC ( tcp, "TCP %p has no route to %s\n",
  273. tcp, sock_ntoa ( peer ) );
  274. rc = -ENETUNREACH;
  275. goto err;
  276. }
  277. tcp->mss = ( mtu - sizeof ( struct tcp_header ) );
  278. /* Bind to local port */
  279. port = tcpip_bind ( st_local, tcp_port_available );
  280. if ( port < 0 ) {
  281. rc = port;
  282. DBGC ( tcp, "TCP %p could not bind: %s\n",
  283. tcp, strerror ( rc ) );
  284. goto err;
  285. }
  286. tcp->local_port = port;
  287. DBGC ( tcp, "TCP %p bound to port %d\n", tcp, tcp->local_port );
  288. /* Start timer to initiate SYN */
  289. start_timer_nodelay ( &tcp->timer );
  290. /* Add a pending operation for the SYN */
  291. pending_get ( &tcp->pending_flags );
  292. /* Attach parent interface, transfer reference to connection
  293. * list and return
  294. */
  295. intf_plug_plug ( &tcp->xfer, xfer );
  296. list_add ( &tcp->list, &tcp_conns );
  297. return 0;
  298. err:
  299. ref_put ( &tcp->refcnt );
  300. return rc;
  301. }
  302. /**
  303. * Close TCP connection
  304. *
  305. * @v tcp TCP connection
  306. * @v rc Reason for close
  307. *
  308. * Closes the data transfer interface. If the TCP state machine is in
  309. * a suitable state, the connection will be deleted.
  310. */
  311. static void tcp_close ( struct tcp_connection *tcp, int rc ) {
  312. struct io_buffer *iobuf;
  313. struct io_buffer *tmp;
  314. /* Close data transfer interface */
  315. intf_shutdown ( &tcp->xfer, rc );
  316. tcp->flags |= TCP_XFER_CLOSED;
  317. /* If we are in CLOSED, or have otherwise not yet received a
  318. * SYN (i.e. we are in LISTEN or SYN_SENT), just delete the
  319. * connection.
  320. */
  321. if ( ! ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) ) {
  322. /* Transition to CLOSED for the sake of debugging messages */
  323. tcp->tcp_state = TCP_CLOSED;
  324. tcp_dump_state ( tcp );
  325. /* Free any unprocessed I/O buffers */
  326. list_for_each_entry_safe ( iobuf, tmp, &tcp->rx_queue, list ) {
  327. list_del ( &iobuf->list );
  328. free_iob ( iobuf );
  329. }
  330. /* Free any unsent I/O buffers */
  331. list_for_each_entry_safe ( iobuf, tmp, &tcp->tx_queue, list ) {
  332. list_del ( &iobuf->list );
  333. free_iob ( iobuf );
  334. pending_put ( &tcp->pending_data );
  335. }
  336. assert ( ! is_pending ( &tcp->pending_data ) );
  337. /* Remove pending operations for SYN and FIN, if applicable */
  338. pending_put ( &tcp->pending_flags );
  339. pending_put ( &tcp->pending_flags );
  340. /* Remove from list and drop reference */
  341. process_del ( &tcp->process );
  342. stop_timer ( &tcp->timer );
  343. stop_timer ( &tcp->wait );
  344. list_del ( &tcp->list );
  345. ref_put ( &tcp->refcnt );
  346. DBGC ( tcp, "TCP %p connection deleted\n", tcp );
  347. return;
  348. }
  349. /* If we have not had our SYN acknowledged (i.e. we are in
  350. * SYN_RCVD), pretend that it has been acknowledged so that we
  351. * can send a FIN without breaking things.
  352. */
  353. if ( ! ( tcp->tcp_state & TCP_STATE_ACKED ( TCP_SYN ) ) )
  354. tcp_rx_ack ( tcp, ( tcp->snd_seq + 1 ), 0 );
  355. /* If we have no data remaining to send, start sending FIN */
  356. if ( list_empty ( &tcp->tx_queue ) &&
  357. ! ( tcp->tcp_state & TCP_STATE_SENT ( TCP_FIN ) ) ) {
  358. tcp->tcp_state |= TCP_STATE_SENT ( TCP_FIN );
  359. tcp_dump_state ( tcp );
  360. process_add ( &tcp->process );
  361. /* Add a pending operation for the FIN */
  362. pending_get ( &tcp->pending_flags );
  363. }
  364. }
  365. /***************************************************************************
  366. *
  367. * Transmit data path
  368. *
  369. ***************************************************************************
  370. */
  371. /**
  372. * Calculate transmission window
  373. *
  374. * @v tcp TCP connection
  375. * @ret len Maximum length that can be sent in a single packet
  376. */
  377. static size_t tcp_xmit_win ( struct tcp_connection *tcp ) {
  378. size_t len;
  379. /* Not ready if we're not in a suitable connection state */
  380. if ( ! TCP_CAN_SEND_DATA ( tcp->tcp_state ) )
  381. return 0;
  382. /* Length is the minimum of the receiver's window and the path MTU */
  383. len = tcp->snd_win;
  384. if ( len > TCP_PATH_MTU )
  385. len = TCP_PATH_MTU;
  386. return len;
  387. }
  388. /**
  389. * Check data-transfer flow control window
  390. *
  391. * @v tcp TCP connection
  392. * @ret len Length of window
  393. */
  394. static size_t tcp_xfer_window ( struct tcp_connection *tcp ) {
  395. /* Not ready if data queue is non-empty. This imposes a limit
  396. * of only one unACKed packet in the TX queue at any time; we
  397. * do this to conserve memory usage.
  398. */
  399. if ( ! list_empty ( &tcp->tx_queue ) )
  400. return 0;
  401. /* Return TCP window length */
  402. return tcp_xmit_win ( tcp );
  403. }
  404. /**
  405. * Find selective acknowledgement block
  406. *
  407. * @v tcp TCP connection
  408. * @v seq SEQ value in SACK block (in host-endian order)
  409. * @v sack SACK block to fill in (in host-endian order)
  410. * @ret len Length of SACK block
  411. */
  412. static uint32_t tcp_sack_block ( struct tcp_connection *tcp, uint32_t seq,
  413. struct tcp_sack_block *sack ) {
  414. struct io_buffer *iobuf;
  415. struct tcp_rx_queued_header *tcpqhdr;
  416. uint32_t left = tcp->rcv_ack;
  417. uint32_t right = left;
  418. /* Find highest block which does not start after SEQ */
  419. list_for_each_entry ( iobuf, &tcp->rx_queue, list ) {
  420. tcpqhdr = iobuf->data;
  421. if ( tcp_cmp ( tcpqhdr->seq, right ) > 0 ) {
  422. if ( tcp_cmp ( tcpqhdr->seq, seq ) > 0 )
  423. break;
  424. left = tcpqhdr->seq;
  425. }
  426. if ( tcp_cmp ( tcpqhdr->nxt, right ) > 0 )
  427. right = tcpqhdr->nxt;
  428. }
  429. /* Fail if this block does not contain SEQ */
  430. if ( tcp_cmp ( right, seq ) < 0 )
  431. return 0;
  432. /* Populate SACK block */
  433. sack->left = left;
  434. sack->right = right;
  435. return ( right - left );
  436. }
  437. /**
  438. * Update TCP selective acknowledgement list
  439. *
  440. * @v tcp TCP connection
  441. * @v seq SEQ value in first SACK block (in host-endian order)
  442. * @ret count Number of SACK blocks
  443. */
  444. static unsigned int tcp_sack ( struct tcp_connection *tcp, uint32_t seq ) {
  445. struct tcp_sack_block sack[TCP_SACK_MAX];
  446. unsigned int old = 0;
  447. unsigned int new = 0;
  448. unsigned int i;
  449. uint32_t len;
  450. /* Populate first new SACK block */
  451. len = tcp_sack_block ( tcp, seq, &sack[0] );
  452. if ( len )
  453. new++;
  454. /* Populate remaining new SACK blocks based on old SACK blocks */
  455. for ( old = 0 ; old < TCP_SACK_MAX ; old++ ) {
  456. /* Stop if we run out of space in the new list */
  457. if ( new == TCP_SACK_MAX )
  458. break;
  459. /* Skip empty old SACK blocks */
  460. if ( tcp->sack[old].left == tcp->sack[old].right )
  461. continue;
  462. /* Populate new SACK block */
  463. len = tcp_sack_block ( tcp, tcp->sack[old].left, &sack[new] );
  464. if ( len == 0 )
  465. continue;
  466. /* Eliminate duplicates */
  467. for ( i = 0 ; i < new ; i++ ) {
  468. if ( sack[i].left == sack[new].left ) {
  469. new--;
  470. break;
  471. }
  472. }
  473. new++;
  474. }
  475. /* Update SACK list */
  476. memset ( tcp->sack, 0, sizeof ( tcp->sack ) );
  477. memcpy ( tcp->sack, sack, ( new * sizeof ( tcp->sack[0] ) ) );
  478. return new;
  479. }
  480. /**
  481. * Process TCP transmit queue
  482. *
  483. * @v tcp TCP connection
  484. * @v max_len Maximum length to process
  485. * @v dest I/O buffer to fill with data, or NULL
  486. * @v remove Remove data from queue
  487. * @ret len Length of data processed
  488. *
  489. * This processes at most @c max_len bytes from the TCP connection's
  490. * transmit queue. Data will be copied into the @c dest I/O buffer
  491. * (if provided) and, if @c remove is true, removed from the transmit
  492. * queue.
  493. */
  494. static size_t tcp_process_tx_queue ( struct tcp_connection *tcp, size_t max_len,
  495. struct io_buffer *dest, int remove ) {
  496. struct io_buffer *iobuf;
  497. struct io_buffer *tmp;
  498. size_t frag_len;
  499. size_t len = 0;
  500. list_for_each_entry_safe ( iobuf, tmp, &tcp->tx_queue, list ) {
  501. frag_len = iob_len ( iobuf );
  502. if ( frag_len > max_len )
  503. frag_len = max_len;
  504. if ( dest ) {
  505. memcpy ( iob_put ( dest, frag_len ), iobuf->data,
  506. frag_len );
  507. }
  508. if ( remove ) {
  509. iob_pull ( iobuf, frag_len );
  510. if ( ! iob_len ( iobuf ) ) {
  511. list_del ( &iobuf->list );
  512. free_iob ( iobuf );
  513. pending_put ( &tcp->pending_data );
  514. }
  515. }
  516. len += frag_len;
  517. max_len -= frag_len;
  518. }
  519. return len;
  520. }
  521. /**
  522. * Transmit any outstanding data (with selective acknowledgement)
  523. *
  524. * @v tcp TCP connection
  525. * @v sack_seq SEQ for first selective acknowledgement (if any)
  526. *
  527. * Transmits any outstanding data on the connection.
  528. *
  529. * Note that even if an error is returned, the retransmission timer
  530. * will have been started if necessary, and so the stack will
  531. * eventually attempt to retransmit the failed packet.
  532. */
  533. static void tcp_xmit_sack ( struct tcp_connection *tcp, uint32_t sack_seq ) {
  534. struct io_buffer *iobuf;
  535. struct tcp_header *tcphdr;
  536. struct tcp_mss_option *mssopt;
  537. struct tcp_window_scale_padded_option *wsopt;
  538. struct tcp_timestamp_padded_option *tsopt;
  539. struct tcp_sack_permitted_padded_option *spopt;
  540. struct tcp_sack_padded_option *sackopt;
  541. struct tcp_sack_block *sack;
  542. void *payload;
  543. unsigned int flags;
  544. unsigned int sack_count;
  545. unsigned int i;
  546. size_t len = 0;
  547. size_t sack_len;
  548. uint32_t seq_len;
  549. uint32_t max_rcv_win;
  550. uint32_t max_representable_win;
  551. int rc;
  552. /* Start profiling */
  553. profile_start ( &tcp_tx_profiler );
  554. /* If retransmission timer is already running, do nothing */
  555. if ( timer_running ( &tcp->timer ) )
  556. return;
  557. /* Calculate both the actual (payload) and sequence space
  558. * lengths that we wish to transmit.
  559. */
  560. if ( TCP_CAN_SEND_DATA ( tcp->tcp_state ) ) {
  561. len = tcp_process_tx_queue ( tcp, tcp_xmit_win ( tcp ),
  562. NULL, 0 );
  563. }
  564. seq_len = len;
  565. flags = TCP_FLAGS_SENDING ( tcp->tcp_state );
  566. if ( flags & ( TCP_SYN | TCP_FIN ) ) {
  567. /* SYN or FIN consume one byte, and we can never send both */
  568. assert ( ! ( ( flags & TCP_SYN ) && ( flags & TCP_FIN ) ) );
  569. seq_len++;
  570. }
  571. tcp->snd_sent = seq_len;
  572. /* If we have nothing to transmit, stop now */
  573. if ( ( seq_len == 0 ) && ! ( tcp->flags & TCP_ACK_PENDING ) )
  574. return;
  575. /* If we are transmitting anything that requires
  576. * acknowledgement (i.e. consumes sequence space), start the
  577. * retransmission timer. Do this before attempting to
  578. * allocate the I/O buffer, in case allocation itself fails.
  579. */
  580. if ( seq_len )
  581. start_timer ( &tcp->timer );
  582. /* Allocate I/O buffer */
  583. iobuf = alloc_iob ( len + TCP_MAX_HEADER_LEN );
  584. if ( ! iobuf ) {
  585. DBGC ( tcp, "TCP %p could not allocate iobuf for %08x..%08x "
  586. "%08x\n", tcp, tcp->snd_seq, ( tcp->snd_seq + seq_len ),
  587. tcp->rcv_ack );
  588. return;
  589. }
  590. iob_reserve ( iobuf, TCP_MAX_HEADER_LEN );
  591. /* Fill data payload from transmit queue */
  592. tcp_process_tx_queue ( tcp, len, iobuf, 0 );
  593. /* Expand receive window if possible */
  594. max_rcv_win = xfer_window ( &tcp->xfer );
  595. if ( max_rcv_win > TCP_MAX_WINDOW_SIZE )
  596. max_rcv_win = TCP_MAX_WINDOW_SIZE;
  597. max_representable_win = ( 0xffff << tcp->rcv_win_scale );
  598. if ( max_rcv_win > max_representable_win )
  599. max_rcv_win = max_representable_win;
  600. max_rcv_win &= ~0x03; /* Keep everything dword-aligned */
  601. if ( tcp->rcv_win < max_rcv_win )
  602. tcp->rcv_win = max_rcv_win;
  603. /* Fill up the TCP header */
  604. payload = iobuf->data;
  605. if ( flags & TCP_SYN ) {
  606. mssopt = iob_push ( iobuf, sizeof ( *mssopt ) );
  607. mssopt->kind = TCP_OPTION_MSS;
  608. mssopt->length = sizeof ( *mssopt );
  609. mssopt->mss = htons ( tcp->mss );
  610. wsopt = iob_push ( iobuf, sizeof ( *wsopt ) );
  611. wsopt->nop = TCP_OPTION_NOP;
  612. wsopt->wsopt.kind = TCP_OPTION_WS;
  613. wsopt->wsopt.length = sizeof ( wsopt->wsopt );
  614. wsopt->wsopt.scale = TCP_RX_WINDOW_SCALE;
  615. spopt = iob_push ( iobuf, sizeof ( *spopt ) );
  616. memset ( spopt->nop, TCP_OPTION_NOP, sizeof ( spopt ) );
  617. spopt->spopt.kind = TCP_OPTION_SACK_PERMITTED;
  618. spopt->spopt.length = sizeof ( spopt->spopt );
  619. }
  620. if ( ( flags & TCP_SYN ) || ( tcp->flags & TCP_TS_ENABLED ) ) {
  621. tsopt = iob_push ( iobuf, sizeof ( *tsopt ) );
  622. memset ( tsopt->nop, TCP_OPTION_NOP, sizeof ( tsopt->nop ) );
  623. tsopt->tsopt.kind = TCP_OPTION_TS;
  624. tsopt->tsopt.length = sizeof ( tsopt->tsopt );
  625. tsopt->tsopt.tsval = htonl ( currticks() );
  626. tsopt->tsopt.tsecr = htonl ( tcp->ts_recent );
  627. }
  628. if ( ( tcp->flags & TCP_SACK_ENABLED ) &&
  629. ( ! list_empty ( &tcp->rx_queue ) ) &&
  630. ( ( sack_count = tcp_sack ( tcp, sack_seq ) ) != 0 ) ) {
  631. sack_len = ( sack_count * sizeof ( *sack ) );
  632. sackopt = iob_push ( iobuf, ( sizeof ( *sackopt ) + sack_len ));
  633. memset ( sackopt->nop, TCP_OPTION_NOP, sizeof ( sackopt->nop ));
  634. sackopt->sackopt.kind = TCP_OPTION_SACK;
  635. sackopt->sackopt.length =
  636. ( sizeof ( sackopt->sackopt ) + sack_len );
  637. sack = ( ( ( void * ) sackopt ) + sizeof ( *sackopt ) );
  638. for ( i = 0 ; i < sack_count ; i++, sack++ ) {
  639. sack->left = htonl ( tcp->sack[i].left );
  640. sack->right = htonl ( tcp->sack[i].right );
  641. }
  642. }
  643. if ( len != 0 )
  644. flags |= TCP_PSH;
  645. tcphdr = iob_push ( iobuf, sizeof ( *tcphdr ) );
  646. memset ( tcphdr, 0, sizeof ( *tcphdr ) );
  647. tcphdr->src = htons ( tcp->local_port );
  648. tcphdr->dest = tcp->peer.st_port;
  649. tcphdr->seq = htonl ( tcp->snd_seq );
  650. tcphdr->ack = htonl ( tcp->rcv_ack );
  651. tcphdr->hlen = ( ( payload - iobuf->data ) << 2 );
  652. tcphdr->flags = flags;
  653. tcphdr->win = htons ( tcp->rcv_win >> tcp->rcv_win_scale );
  654. tcphdr->csum = tcpip_chksum ( iobuf->data, iob_len ( iobuf ) );
  655. /* Dump header */
  656. DBGC2 ( tcp, "TCP %p TX %d->%d %08x..%08x %08x %4zd",
  657. tcp, ntohs ( tcphdr->src ), ntohs ( tcphdr->dest ),
  658. ntohl ( tcphdr->seq ), ( ntohl ( tcphdr->seq ) + seq_len ),
  659. ntohl ( tcphdr->ack ), len );
  660. tcp_dump_flags ( tcp, tcphdr->flags );
  661. DBGC2 ( tcp, "\n" );
  662. /* Transmit packet */
  663. if ( ( rc = tcpip_tx ( iobuf, &tcp_protocol, NULL, &tcp->peer, NULL,
  664. &tcphdr->csum ) ) != 0 ) {
  665. DBGC ( tcp, "TCP %p could not transmit %08x..%08x %08x: %s\n",
  666. tcp, tcp->snd_seq, ( tcp->snd_seq + tcp->snd_sent ),
  667. tcp->rcv_ack, strerror ( rc ) );
  668. return;
  669. }
  670. /* Clear ACK-pending flag */
  671. tcp->flags &= ~TCP_ACK_PENDING;
  672. profile_stop ( &tcp_tx_profiler );
  673. }
  674. /**
  675. * Transmit any outstanding data
  676. *
  677. * @v tcp TCP connection
  678. */
  679. static void tcp_xmit ( struct tcp_connection *tcp ) {
  680. /* Transmit without an explicit first SACK */
  681. tcp_xmit_sack ( tcp, tcp->rcv_ack );
  682. }
  683. /** TCP process descriptor */
  684. static struct process_descriptor tcp_process_desc =
  685. PROC_DESC_ONCE ( struct tcp_connection, process, tcp_xmit );
  686. /**
  687. * Retransmission timer expired
  688. *
  689. * @v timer Retransmission timer
  690. * @v over Failure indicator
  691. */
  692. static void tcp_expired ( struct retry_timer *timer, int over ) {
  693. struct tcp_connection *tcp =
  694. container_of ( timer, struct tcp_connection, timer );
  695. DBGC ( tcp, "TCP %p timer %s in %s for %08x..%08x %08x\n", tcp,
  696. ( over ? "expired" : "fired" ), tcp_state ( tcp->tcp_state ),
  697. tcp->snd_seq, ( tcp->snd_seq + tcp->snd_sent ), tcp->rcv_ack );
  698. assert ( ( tcp->tcp_state == TCP_SYN_SENT ) ||
  699. ( tcp->tcp_state == TCP_SYN_RCVD ) ||
  700. ( tcp->tcp_state == TCP_ESTABLISHED ) ||
  701. ( tcp->tcp_state == TCP_FIN_WAIT_1 ) ||
  702. ( tcp->tcp_state == TCP_CLOSE_WAIT ) ||
  703. ( tcp->tcp_state == TCP_CLOSING_OR_LAST_ACK ) );
  704. if ( over ) {
  705. /* If we have finally timed out and given up,
  706. * terminate the connection
  707. */
  708. tcp->tcp_state = TCP_CLOSED;
  709. tcp_dump_state ( tcp );
  710. tcp_close ( tcp, -ETIMEDOUT );
  711. } else {
  712. /* Otherwise, retransmit the packet */
  713. tcp_xmit ( tcp );
  714. }
  715. }
  716. /**
  717. * Shutdown timer expired
  718. *
  719. * @v timer Shutdown timer
  720. * @v over Failure indicator
  721. */
  722. static void tcp_wait_expired ( struct retry_timer *timer, int over __unused ) {
  723. struct tcp_connection *tcp =
  724. container_of ( timer, struct tcp_connection, wait );
  725. assert ( tcp->tcp_state == TCP_TIME_WAIT );
  726. DBGC ( tcp, "TCP %p wait complete in %s for %08x..%08x %08x\n", tcp,
  727. tcp_state ( tcp->tcp_state ), tcp->snd_seq,
  728. ( tcp->snd_seq + tcp->snd_sent ), tcp->rcv_ack );
  729. tcp->tcp_state = TCP_CLOSED;
  730. tcp_dump_state ( tcp );
  731. tcp_close ( tcp, 0 );
  732. }
  733. /**
  734. * Send RST response to incoming packet
  735. *
  736. * @v in_tcphdr TCP header of incoming packet
  737. * @ret rc Return status code
  738. */
  739. static int tcp_xmit_reset ( struct tcp_connection *tcp,
  740. struct sockaddr_tcpip *st_dest,
  741. struct tcp_header *in_tcphdr ) {
  742. struct io_buffer *iobuf;
  743. struct tcp_header *tcphdr;
  744. int rc;
  745. /* Allocate space for dataless TX buffer */
  746. iobuf = alloc_iob ( TCP_MAX_HEADER_LEN );
  747. if ( ! iobuf ) {
  748. DBGC ( tcp, "TCP %p could not allocate iobuf for RST "
  749. "%08x..%08x %08x\n", tcp, ntohl ( in_tcphdr->ack ),
  750. ntohl ( in_tcphdr->ack ), ntohl ( in_tcphdr->seq ) );
  751. return -ENOMEM;
  752. }
  753. iob_reserve ( iobuf, TCP_MAX_HEADER_LEN );
  754. /* Construct RST response */
  755. tcphdr = iob_push ( iobuf, sizeof ( *tcphdr ) );
  756. memset ( tcphdr, 0, sizeof ( *tcphdr ) );
  757. tcphdr->src = in_tcphdr->dest;
  758. tcphdr->dest = in_tcphdr->src;
  759. tcphdr->seq = in_tcphdr->ack;
  760. tcphdr->ack = in_tcphdr->seq;
  761. tcphdr->hlen = ( ( sizeof ( *tcphdr ) / 4 ) << 4 );
  762. tcphdr->flags = ( TCP_RST | TCP_ACK );
  763. tcphdr->win = htons ( 0 );
  764. tcphdr->csum = tcpip_chksum ( iobuf->data, iob_len ( iobuf ) );
  765. /* Dump header */
  766. DBGC2 ( tcp, "TCP %p TX %d->%d %08x..%08x %08x %4d",
  767. tcp, ntohs ( tcphdr->src ), ntohs ( tcphdr->dest ),
  768. ntohl ( tcphdr->seq ), ( ntohl ( tcphdr->seq ) ),
  769. ntohl ( tcphdr->ack ), 0 );
  770. tcp_dump_flags ( tcp, tcphdr->flags );
  771. DBGC2 ( tcp, "\n" );
  772. /* Transmit packet */
  773. if ( ( rc = tcpip_tx ( iobuf, &tcp_protocol, NULL, st_dest,
  774. NULL, &tcphdr->csum ) ) != 0 ) {
  775. DBGC ( tcp, "TCP %p could not transmit RST %08x..%08x %08x: "
  776. "%s\n", tcp, ntohl ( in_tcphdr->ack ),
  777. ntohl ( in_tcphdr->ack ), ntohl ( in_tcphdr->seq ),
  778. strerror ( rc ) );
  779. return rc;
  780. }
  781. return 0;
  782. }
  783. /***************************************************************************
  784. *
  785. * Receive data path
  786. *
  787. ***************************************************************************
  788. */
  789. /**
  790. * Identify TCP connection by local port number
  791. *
  792. * @v local_port Local port
  793. * @ret tcp TCP connection, or NULL
  794. */
  795. static struct tcp_connection * tcp_demux ( unsigned int local_port ) {
  796. struct tcp_connection *tcp;
  797. list_for_each_entry ( tcp, &tcp_conns, list ) {
  798. if ( tcp->local_port == local_port )
  799. return tcp;
  800. }
  801. return NULL;
  802. }
  803. /**
  804. * Parse TCP received options
  805. *
  806. * @v tcp TCP connection
  807. * @v data Raw options data
  808. * @v len Raw options length
  809. * @v options Options structure to fill in
  810. */
  811. static void tcp_rx_opts ( struct tcp_connection *tcp, const void *data,
  812. size_t len, struct tcp_options *options ) {
  813. const void *end = ( data + len );
  814. const struct tcp_option *option;
  815. unsigned int kind;
  816. memset ( options, 0, sizeof ( *options ) );
  817. while ( data < end ) {
  818. option = data;
  819. kind = option->kind;
  820. if ( kind == TCP_OPTION_END )
  821. return;
  822. if ( kind == TCP_OPTION_NOP ) {
  823. data++;
  824. continue;
  825. }
  826. switch ( kind ) {
  827. case TCP_OPTION_MSS:
  828. options->mssopt = data;
  829. break;
  830. case TCP_OPTION_WS:
  831. options->wsopt = data;
  832. break;
  833. case TCP_OPTION_SACK_PERMITTED:
  834. options->spopt = data;
  835. break;
  836. case TCP_OPTION_SACK:
  837. /* Ignore received SACKs */
  838. break;
  839. case TCP_OPTION_TS:
  840. options->tsopt = data;
  841. break;
  842. default:
  843. DBGC ( tcp, "TCP %p received unknown option %d\n",
  844. tcp, kind );
  845. break;
  846. }
  847. data += option->length;
  848. }
  849. }
  850. /**
  851. * Consume received sequence space
  852. *
  853. * @v tcp TCP connection
  854. * @v seq_len Sequence space length to consume
  855. */
  856. static void tcp_rx_seq ( struct tcp_connection *tcp, uint32_t seq_len ) {
  857. unsigned int sack;
  858. /* Sanity check */
  859. assert ( seq_len > 0 );
  860. /* Update acknowledgement number */
  861. tcp->rcv_ack += seq_len;
  862. /* Update window */
  863. if ( tcp->rcv_win > seq_len ) {
  864. tcp->rcv_win -= seq_len;
  865. } else {
  866. tcp->rcv_win = 0;
  867. }
  868. /* Update timestamp */
  869. tcp->ts_recent = tcp->ts_val;
  870. /* Update SACK list */
  871. for ( sack = 0 ; sack < TCP_SACK_MAX ; sack++ ) {
  872. if ( tcp->sack[sack].left == tcp->sack[sack].right )
  873. continue;
  874. if ( tcp_cmp ( tcp->sack[sack].left, tcp->rcv_ack ) < 0 )
  875. tcp->sack[sack].left = tcp->rcv_ack;
  876. if ( tcp_cmp ( tcp->sack[sack].right, tcp->rcv_ack ) < 0 )
  877. tcp->sack[sack].right = tcp->rcv_ack;
  878. }
  879. /* Mark ACK as pending */
  880. tcp->flags |= TCP_ACK_PENDING;
  881. }
  882. /**
  883. * Handle TCP received SYN
  884. *
  885. * @v tcp TCP connection
  886. * @v seq SEQ value (in host-endian order)
  887. * @v options TCP options
  888. * @ret rc Return status code
  889. */
  890. static int tcp_rx_syn ( struct tcp_connection *tcp, uint32_t seq,
  891. struct tcp_options *options ) {
  892. /* Synchronise sequence numbers on first SYN */
  893. if ( ! ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) ) {
  894. tcp->rcv_ack = seq;
  895. if ( options->tsopt )
  896. tcp->flags |= TCP_TS_ENABLED;
  897. if ( options->spopt )
  898. tcp->flags |= TCP_SACK_ENABLED;
  899. if ( options->wsopt ) {
  900. tcp->snd_win_scale = options->wsopt->scale;
  901. tcp->rcv_win_scale = TCP_RX_WINDOW_SCALE;
  902. }
  903. }
  904. /* Ignore duplicate SYN */
  905. if ( seq != tcp->rcv_ack )
  906. return 0;
  907. /* Acknowledge SYN */
  908. tcp_rx_seq ( tcp, 1 );
  909. /* Mark SYN as received and start sending ACKs with each packet */
  910. tcp->tcp_state |= ( TCP_STATE_SENT ( TCP_ACK ) |
  911. TCP_STATE_RCVD ( TCP_SYN ) );
  912. return 0;
  913. }
  914. /**
  915. * Handle TCP received ACK
  916. *
  917. * @v tcp TCP connection
  918. * @v ack ACK value (in host-endian order)
  919. * @v win WIN value (in host-endian order)
  920. * @ret rc Return status code
  921. */
  922. static int tcp_rx_ack ( struct tcp_connection *tcp, uint32_t ack,
  923. uint32_t win ) {
  924. uint32_t ack_len = ( ack - tcp->snd_seq );
  925. size_t len;
  926. unsigned int acked_flags;
  927. /* Check for out-of-range or old duplicate ACKs */
  928. if ( ack_len > tcp->snd_sent ) {
  929. DBGC ( tcp, "TCP %p received ACK for %08x..%08x, "
  930. "sent only %08x..%08x\n", tcp, tcp->snd_seq,
  931. ( tcp->snd_seq + ack_len ), tcp->snd_seq,
  932. ( tcp->snd_seq + tcp->snd_sent ) );
  933. if ( TCP_HAS_BEEN_ESTABLISHED ( tcp->tcp_state ) ) {
  934. /* Just ignore what might be old duplicate ACKs */
  935. return 0;
  936. } else {
  937. /* Send RST if an out-of-range ACK is received
  938. * on a not-yet-established connection, as per
  939. * RFC 793.
  940. */
  941. return -EINVAL;
  942. }
  943. }
  944. /* Update window size */
  945. tcp->snd_win = win;
  946. /* Ignore ACKs that don't actually acknowledge any new data.
  947. * (In particular, do not stop the retransmission timer; this
  948. * avoids creating a sorceror's apprentice syndrome when a
  949. * duplicate ACK is received and we still have data in our
  950. * transmit queue.)
  951. */
  952. if ( ack_len == 0 )
  953. return 0;
  954. /* Stop the retransmission timer */
  955. stop_timer ( &tcp->timer );
  956. /* Determine acknowledged flags and data length */
  957. len = ack_len;
  958. acked_flags = ( TCP_FLAGS_SENDING ( tcp->tcp_state ) &
  959. ( TCP_SYN | TCP_FIN ) );
  960. if ( acked_flags ) {
  961. len--;
  962. pending_put ( &tcp->pending_flags );
  963. }
  964. /* Update SEQ and sent counters */
  965. tcp->snd_seq = ack;
  966. tcp->snd_sent = 0;
  967. /* Remove any acknowledged data from transmit queue */
  968. tcp_process_tx_queue ( tcp, len, NULL, 1 );
  969. /* Mark SYN/FIN as acknowledged if applicable. */
  970. if ( acked_flags )
  971. tcp->tcp_state |= TCP_STATE_ACKED ( acked_flags );
  972. /* Start sending FIN if we've had all possible data ACKed */
  973. if ( list_empty ( &tcp->tx_queue ) &&
  974. ( tcp->flags & TCP_XFER_CLOSED ) &&
  975. ! ( tcp->tcp_state & TCP_STATE_SENT ( TCP_FIN ) ) ) {
  976. tcp->tcp_state |= TCP_STATE_SENT ( TCP_FIN );
  977. pending_get ( &tcp->pending_flags );
  978. }
  979. return 0;
  980. }
  981. /**
  982. * Handle TCP received data
  983. *
  984. * @v tcp TCP connection
  985. * @v seq SEQ value (in host-endian order)
  986. * @v iobuf I/O buffer
  987. * @ret rc Return status code
  988. *
  989. * This function takes ownership of the I/O buffer.
  990. */
  991. static int tcp_rx_data ( struct tcp_connection *tcp, uint32_t seq,
  992. struct io_buffer *iobuf ) {
  993. uint32_t already_rcvd;
  994. uint32_t len;
  995. int rc;
  996. /* Ignore duplicate or out-of-order data */
  997. already_rcvd = ( tcp->rcv_ack - seq );
  998. len = iob_len ( iobuf );
  999. if ( already_rcvd >= len ) {
  1000. free_iob ( iobuf );
  1001. return 0;
  1002. }
  1003. iob_pull ( iobuf, already_rcvd );
  1004. len -= already_rcvd;
  1005. /* Acknowledge new data */
  1006. tcp_rx_seq ( tcp, len );
  1007. /* Deliver data to application */
  1008. profile_start ( &tcp_xfer_profiler );
  1009. if ( ( rc = xfer_deliver_iob ( &tcp->xfer, iobuf ) ) != 0 ) {
  1010. DBGC ( tcp, "TCP %p could not deliver %08x..%08x: %s\n",
  1011. tcp, seq, ( seq + len ), strerror ( rc ) );
  1012. return rc;
  1013. }
  1014. profile_stop ( &tcp_xfer_profiler );
  1015. return 0;
  1016. }
  1017. /**
  1018. * Handle TCP received FIN
  1019. *
  1020. * @v tcp TCP connection
  1021. * @v seq SEQ value (in host-endian order)
  1022. * @ret rc Return status code
  1023. */
  1024. static int tcp_rx_fin ( struct tcp_connection *tcp, uint32_t seq ) {
  1025. /* Ignore duplicate or out-of-order FIN */
  1026. if ( seq != tcp->rcv_ack )
  1027. return 0;
  1028. /* Acknowledge FIN */
  1029. tcp_rx_seq ( tcp, 1 );
  1030. /* Mark FIN as received */
  1031. tcp->tcp_state |= TCP_STATE_RCVD ( TCP_FIN );
  1032. /* Close connection */
  1033. tcp_close ( tcp, 0 );
  1034. return 0;
  1035. }
  1036. /**
  1037. * Handle TCP received RST
  1038. *
  1039. * @v tcp TCP connection
  1040. * @v seq SEQ value (in host-endian order)
  1041. * @ret rc Return status code
  1042. */
  1043. static int tcp_rx_rst ( struct tcp_connection *tcp, uint32_t seq ) {
  1044. /* Accept RST only if it falls within the window. If we have
  1045. * not yet received a SYN, then we have no window to test
  1046. * against, so fall back to checking that our SYN has been
  1047. * ACKed.
  1048. */
  1049. if ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) {
  1050. if ( ! tcp_in_window ( seq, tcp->rcv_ack, tcp->rcv_win ) )
  1051. return 0;
  1052. } else {
  1053. if ( ! ( tcp->tcp_state & TCP_STATE_ACKED ( TCP_SYN ) ) )
  1054. return 0;
  1055. }
  1056. /* Abort connection */
  1057. tcp->tcp_state = TCP_CLOSED;
  1058. tcp_dump_state ( tcp );
  1059. tcp_close ( tcp, -ECONNRESET );
  1060. DBGC ( tcp, "TCP %p connection reset by peer\n", tcp );
  1061. return -ECONNRESET;
  1062. }
  1063. /**
  1064. * Enqueue received TCP packet
  1065. *
  1066. * @v tcp TCP connection
  1067. * @v seq SEQ value (in host-endian order)
  1068. * @v flags TCP flags
  1069. * @v iobuf I/O buffer
  1070. */
  1071. static void tcp_rx_enqueue ( struct tcp_connection *tcp, uint32_t seq,
  1072. uint8_t flags, struct io_buffer *iobuf ) {
  1073. struct tcp_rx_queued_header *tcpqhdr;
  1074. struct io_buffer *queued;
  1075. size_t len;
  1076. uint32_t seq_len;
  1077. uint32_t nxt;
  1078. /* Calculate remaining flags and sequence length. Note that
  1079. * SYN, if present, has already been processed by this point.
  1080. */
  1081. flags &= TCP_FIN;
  1082. len = iob_len ( iobuf );
  1083. seq_len = ( len + ( flags ? 1 : 0 ) );
  1084. nxt = ( seq + seq_len );
  1085. /* Discard immediately (to save memory) if:
  1086. *
  1087. * a) we have not yet received a SYN (and so have no defined
  1088. * receive window), or
  1089. * b) the packet lies entirely outside the receive window, or
  1090. * c) there is no further content to process.
  1091. */
  1092. if ( ( ! ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) ) ||
  1093. ( tcp_cmp ( seq, tcp->rcv_ack + tcp->rcv_win ) >= 0 ) ||
  1094. ( tcp_cmp ( nxt, tcp->rcv_ack ) < 0 ) ||
  1095. ( seq_len == 0 ) ) {
  1096. free_iob ( iobuf );
  1097. return;
  1098. }
  1099. /* Add internal header */
  1100. tcpqhdr = iob_push ( iobuf, sizeof ( *tcpqhdr ) );
  1101. tcpqhdr->seq = seq;
  1102. tcpqhdr->nxt = nxt;
  1103. tcpqhdr->flags = flags;
  1104. /* Add to RX queue */
  1105. list_for_each_entry ( queued, &tcp->rx_queue, list ) {
  1106. tcpqhdr = queued->data;
  1107. if ( tcp_cmp ( seq, tcpqhdr->seq ) < 0 )
  1108. break;
  1109. }
  1110. list_add_tail ( &iobuf->list, &queued->list );
  1111. }
  1112. /**
  1113. * Process receive queue
  1114. *
  1115. * @v tcp TCP connection
  1116. */
  1117. static void tcp_process_rx_queue ( struct tcp_connection *tcp ) {
  1118. struct io_buffer *iobuf;
  1119. struct tcp_rx_queued_header *tcpqhdr;
  1120. uint32_t seq;
  1121. unsigned int flags;
  1122. size_t len;
  1123. /* Process all applicable received buffers. Note that we
  1124. * cannot use list_for_each_entry() to iterate over the RX
  1125. * queue, since tcp_discard() may remove packets from the RX
  1126. * queue while we are processing.
  1127. */
  1128. while ( ( iobuf = list_first_entry ( &tcp->rx_queue, struct io_buffer,
  1129. list ) ) ) {
  1130. /* Stop processing when we hit the first gap */
  1131. tcpqhdr = iobuf->data;
  1132. if ( tcp_cmp ( tcpqhdr->seq, tcp->rcv_ack ) > 0 )
  1133. break;
  1134. /* Strip internal header and remove from RX queue */
  1135. list_del ( &iobuf->list );
  1136. seq = tcpqhdr->seq;
  1137. flags = tcpqhdr->flags;
  1138. iob_pull ( iobuf, sizeof ( *tcpqhdr ) );
  1139. len = iob_len ( iobuf );
  1140. /* Handle new data, if any */
  1141. tcp_rx_data ( tcp, seq, iob_disown ( iobuf ) );
  1142. seq += len;
  1143. /* Handle FIN, if present */
  1144. if ( flags & TCP_FIN ) {
  1145. tcp_rx_fin ( tcp, seq );
  1146. seq++;
  1147. }
  1148. }
  1149. }
  1150. /**
  1151. * Process received packet
  1152. *
  1153. * @v iobuf I/O buffer
  1154. * @v netdev Network device
  1155. * @v st_src Partially-filled source address
  1156. * @v st_dest Partially-filled destination address
  1157. * @v pshdr_csum Pseudo-header checksum
  1158. * @ret rc Return status code
  1159. */
  1160. static int tcp_rx ( struct io_buffer *iobuf,
  1161. struct net_device *netdev __unused,
  1162. struct sockaddr_tcpip *st_src,
  1163. struct sockaddr_tcpip *st_dest __unused,
  1164. uint16_t pshdr_csum ) {
  1165. struct tcp_header *tcphdr = iobuf->data;
  1166. struct tcp_connection *tcp;
  1167. struct tcp_options options;
  1168. size_t hlen;
  1169. uint16_t csum;
  1170. uint32_t seq;
  1171. uint32_t ack;
  1172. uint16_t raw_win;
  1173. uint32_t win;
  1174. unsigned int flags;
  1175. size_t len;
  1176. uint32_t seq_len;
  1177. size_t old_xfer_window;
  1178. int rc;
  1179. /* Start profiling */
  1180. profile_start ( &tcp_rx_profiler );
  1181. /* Sanity check packet */
  1182. if ( iob_len ( iobuf ) < sizeof ( *tcphdr ) ) {
  1183. DBG ( "TCP packet too short at %zd bytes (min %zd bytes)\n",
  1184. iob_len ( iobuf ), sizeof ( *tcphdr ) );
  1185. rc = -EINVAL;
  1186. goto discard;
  1187. }
  1188. hlen = ( ( tcphdr->hlen & TCP_MASK_HLEN ) / 16 ) * 4;
  1189. if ( hlen < sizeof ( *tcphdr ) ) {
  1190. DBG ( "TCP header too short at %zd bytes (min %zd bytes)\n",
  1191. hlen, sizeof ( *tcphdr ) );
  1192. rc = -EINVAL;
  1193. goto discard;
  1194. }
  1195. if ( hlen > iob_len ( iobuf ) ) {
  1196. DBG ( "TCP header too long at %zd bytes (max %zd bytes)\n",
  1197. hlen, iob_len ( iobuf ) );
  1198. rc = -EINVAL;
  1199. goto discard;
  1200. }
  1201. csum = tcpip_continue_chksum ( pshdr_csum, iobuf->data,
  1202. iob_len ( iobuf ) );
  1203. if ( csum != 0 ) {
  1204. DBG ( "TCP checksum incorrect (is %04x including checksum "
  1205. "field, should be 0000)\n", csum );
  1206. rc = -EINVAL;
  1207. goto discard;
  1208. }
  1209. /* Parse parameters from header and strip header */
  1210. tcp = tcp_demux ( ntohs ( tcphdr->dest ) );
  1211. seq = ntohl ( tcphdr->seq );
  1212. ack = ntohl ( tcphdr->ack );
  1213. raw_win = ntohs ( tcphdr->win );
  1214. flags = tcphdr->flags;
  1215. tcp_rx_opts ( tcp, ( ( ( void * ) tcphdr ) + sizeof ( *tcphdr ) ),
  1216. ( hlen - sizeof ( *tcphdr ) ), &options );
  1217. if ( tcp && options.tsopt )
  1218. tcp->ts_val = ntohl ( options.tsopt->tsval );
  1219. iob_pull ( iobuf, hlen );
  1220. len = iob_len ( iobuf );
  1221. seq_len = ( len + ( ( flags & TCP_SYN ) ? 1 : 0 ) +
  1222. ( ( flags & TCP_FIN ) ? 1 : 0 ) );
  1223. /* Dump header */
  1224. DBGC2 ( tcp, "TCP %p RX %d<-%d %08x %08x..%08x %4zd",
  1225. tcp, ntohs ( tcphdr->dest ), ntohs ( tcphdr->src ),
  1226. ntohl ( tcphdr->ack ), ntohl ( tcphdr->seq ),
  1227. ( ntohl ( tcphdr->seq ) + seq_len ), len );
  1228. tcp_dump_flags ( tcp, tcphdr->flags );
  1229. DBGC2 ( tcp, "\n" );
  1230. /* If no connection was found, silently drop packet */
  1231. if ( ! tcp ) {
  1232. rc = -ENOTCONN;
  1233. goto discard;
  1234. }
  1235. /* Record old data-transfer window */
  1236. old_xfer_window = tcp_xfer_window ( tcp );
  1237. /* Handle ACK, if present */
  1238. if ( flags & TCP_ACK ) {
  1239. win = ( raw_win << tcp->snd_win_scale );
  1240. if ( ( rc = tcp_rx_ack ( tcp, ack, win ) ) != 0 ) {
  1241. tcp_xmit_reset ( tcp, st_src, tcphdr );
  1242. goto discard;
  1243. }
  1244. }
  1245. /* Force an ACK if this packet is out of order */
  1246. if ( ( tcp->tcp_state & TCP_STATE_RCVD ( TCP_SYN ) ) &&
  1247. ( seq != tcp->rcv_ack ) ) {
  1248. tcp->flags |= TCP_ACK_PENDING;
  1249. }
  1250. /* Handle SYN, if present */
  1251. if ( flags & TCP_SYN ) {
  1252. tcp_rx_syn ( tcp, seq, &options );
  1253. seq++;
  1254. }
  1255. /* Handle RST, if present */
  1256. if ( flags & TCP_RST ) {
  1257. if ( ( rc = tcp_rx_rst ( tcp, seq ) ) != 0 )
  1258. goto discard;
  1259. }
  1260. /* Enqueue received data */
  1261. tcp_rx_enqueue ( tcp, seq, flags, iob_disown ( iobuf ) );
  1262. /* Process receive queue */
  1263. tcp_process_rx_queue ( tcp );
  1264. /* Dump out any state change as a result of the received packet */
  1265. tcp_dump_state ( tcp );
  1266. /* Schedule transmission of ACK (and any pending data). If we
  1267. * have received any out-of-order packets (i.e. if the receive
  1268. * queue remains non-empty after processing) then send the ACK
  1269. * immediately in order to trigger Fast Retransmission.
  1270. */
  1271. if ( list_empty ( &tcp->rx_queue ) ) {
  1272. process_add ( &tcp->process );
  1273. } else {
  1274. tcp_xmit_sack ( tcp, seq );
  1275. }
  1276. /* If this packet was the last we expect to receive, set up
  1277. * timer to expire and cause the connection to be freed.
  1278. */
  1279. if ( TCP_CLOSED_GRACEFULLY ( tcp->tcp_state ) ) {
  1280. stop_timer ( &tcp->wait );
  1281. start_timer_fixed ( &tcp->wait, ( 2 * TCP_MSL ) );
  1282. }
  1283. /* Notify application if window has changed */
  1284. if ( tcp_xfer_window ( tcp ) != old_xfer_window )
  1285. xfer_window_changed ( &tcp->xfer );
  1286. profile_stop ( &tcp_rx_profiler );
  1287. return 0;
  1288. discard:
  1289. /* Free received packet */
  1290. free_iob ( iobuf );
  1291. return rc;
  1292. }
  1293. /** TCP protocol */
  1294. struct tcpip_protocol tcp_protocol __tcpip_protocol = {
  1295. .name = "TCP",
  1296. .rx = tcp_rx,
  1297. .tcpip_proto = IP_TCP,
  1298. };
  1299. /**
  1300. * Discard some cached TCP data
  1301. *
  1302. * @ret discarded Number of cached items discarded
  1303. */
  1304. static unsigned int tcp_discard ( void ) {
  1305. struct tcp_connection *tcp;
  1306. struct io_buffer *iobuf;
  1307. unsigned int discarded = 0;
  1308. /* Try to drop one queued RX packet from each connection */
  1309. list_for_each_entry ( tcp, &tcp_conns, list ) {
  1310. list_for_each_entry_reverse ( iobuf, &tcp->rx_queue, list ) {
  1311. /* Remove packet from queue */
  1312. list_del ( &iobuf->list );
  1313. free_iob ( iobuf );
  1314. /* Report discard */
  1315. discarded++;
  1316. break;
  1317. }
  1318. }
  1319. return discarded;
  1320. }
  1321. /** TCP cache discarder */
  1322. struct cache_discarder tcp_discarder __cache_discarder ( CACHE_NORMAL ) = {
  1323. .discard = tcp_discard,
  1324. };
  1325. /**
  1326. * Find first TCP connection that has not yet been closed
  1327. *
  1328. * @ret tcp First unclosed connection, or NULL
  1329. */
  1330. static struct tcp_connection * tcp_first_unclosed ( void ) {
  1331. struct tcp_connection *tcp;
  1332. /* Find first connection which has not yet been closed */
  1333. list_for_each_entry ( tcp, &tcp_conns, list ) {
  1334. if ( ! ( tcp->flags & TCP_XFER_CLOSED ) )
  1335. return tcp;
  1336. }
  1337. return NULL;
  1338. }
  1339. /**
  1340. * Find first TCP connection that has not yet finished all operations
  1341. *
  1342. * @ret tcp First unfinished connection, or NULL
  1343. */
  1344. static struct tcp_connection * tcp_first_unfinished ( void ) {
  1345. struct tcp_connection *tcp;
  1346. /* Find first connection which has not yet closed gracefully,
  1347. * or which still has a pending transmission (e.g. to ACK the
  1348. * received FIN).
  1349. */
  1350. list_for_each_entry ( tcp, &tcp_conns, list ) {
  1351. if ( ( ! TCP_CLOSED_GRACEFULLY ( tcp->tcp_state ) ) ||
  1352. process_running ( &tcp->process ) ) {
  1353. return tcp;
  1354. }
  1355. }
  1356. return NULL;
  1357. }
  1358. /**
  1359. * Shut down all TCP connections
  1360. *
  1361. */
  1362. static void tcp_shutdown ( int booting __unused ) {
  1363. struct tcp_connection *tcp;
  1364. unsigned long start;
  1365. unsigned long elapsed;
  1366. /* Initiate a graceful close of all connections, allowing for
  1367. * the fact that the connection list may change as we do so.
  1368. */
  1369. while ( ( tcp = tcp_first_unclosed() ) ) {
  1370. DBGC ( tcp, "TCP %p closing for shutdown\n", tcp );
  1371. tcp_close ( tcp, -ECANCELED );
  1372. }
  1373. /* Wait for all connections to finish closing gracefully */
  1374. start = currticks();
  1375. while ( ( tcp = tcp_first_unfinished() ) &&
  1376. ( ( elapsed = ( currticks() - start ) ) < TCP_FINISH_TIMEOUT )){
  1377. step();
  1378. }
  1379. /* Forcibly close any remaining connections */
  1380. while ( ( tcp = list_first_entry ( &tcp_conns, struct tcp_connection,
  1381. list ) ) != NULL ) {
  1382. tcp->tcp_state = TCP_CLOSED;
  1383. tcp_dump_state ( tcp );
  1384. tcp_close ( tcp, -ECANCELED );
  1385. }
  1386. }
  1387. /** TCP shutdown function */
  1388. struct startup_fn tcp_startup_fn __startup_fn ( STARTUP_LATE ) = {
  1389. .shutdown = tcp_shutdown,
  1390. };
  1391. /***************************************************************************
  1392. *
  1393. * Data transfer interface
  1394. *
  1395. ***************************************************************************
  1396. */
  1397. /**
  1398. * Close interface
  1399. *
  1400. * @v tcp TCP connection
  1401. * @v rc Reason for close
  1402. */
  1403. static void tcp_xfer_close ( struct tcp_connection *tcp, int rc ) {
  1404. /* Close data transfer interface */
  1405. tcp_close ( tcp, rc );
  1406. /* Transmit FIN, if possible */
  1407. tcp_xmit ( tcp );
  1408. }
  1409. /**
  1410. * Deliver datagram as I/O buffer
  1411. *
  1412. * @v tcp TCP connection
  1413. * @v iobuf Datagram I/O buffer
  1414. * @v meta Data transfer metadata
  1415. * @ret rc Return status code
  1416. */
  1417. static int tcp_xfer_deliver ( struct tcp_connection *tcp,
  1418. struct io_buffer *iobuf,
  1419. struct xfer_metadata *meta __unused ) {
  1420. /* Enqueue packet */
  1421. list_add_tail ( &iobuf->list, &tcp->tx_queue );
  1422. /* Each enqueued packet is a pending operation */
  1423. pending_get ( &tcp->pending_data );
  1424. /* Transmit data, if possible */
  1425. tcp_xmit ( tcp );
  1426. return 0;
  1427. }
  1428. /** TCP data transfer interface operations */
  1429. static struct interface_operation tcp_xfer_operations[] = {
  1430. INTF_OP ( xfer_deliver, struct tcp_connection *, tcp_xfer_deliver ),
  1431. INTF_OP ( xfer_window, struct tcp_connection *, tcp_xfer_window ),
  1432. INTF_OP ( intf_close, struct tcp_connection *, tcp_xfer_close ),
  1433. };
  1434. /** TCP data transfer interface descriptor */
  1435. static struct interface_descriptor tcp_xfer_desc =
  1436. INTF_DESC ( struct tcp_connection, xfer, tcp_xfer_operations );
  1437. /***************************************************************************
  1438. *
  1439. * Openers
  1440. *
  1441. ***************************************************************************
  1442. */
  1443. /** TCP IPv4 socket opener */
  1444. struct socket_opener tcp_ipv4_socket_opener __socket_opener = {
  1445. .semantics = TCP_SOCK_STREAM,
  1446. .family = AF_INET,
  1447. .open = tcp_open,
  1448. };
  1449. /** TCP IPv6 socket opener */
  1450. struct socket_opener tcp_ipv6_socket_opener __socket_opener = {
  1451. .semantics = TCP_SOCK_STREAM,
  1452. .family = AF_INET6,
  1453. .open = tcp_open,
  1454. };
  1455. /** Linkage hack */
  1456. int tcp_sock_stream = TCP_SOCK_STREAM;
  1457. /**
  1458. * Open TCP URI
  1459. *
  1460. * @v xfer Data transfer interface
  1461. * @v uri URI
  1462. * @ret rc Return status code
  1463. */
  1464. static int tcp_open_uri ( struct interface *xfer, struct uri *uri ) {
  1465. struct sockaddr_tcpip peer;
  1466. /* Sanity check */
  1467. if ( ! uri->host )
  1468. return -EINVAL;
  1469. memset ( &peer, 0, sizeof ( peer ) );
  1470. peer.st_port = htons ( uri_port ( uri, 0 ) );
  1471. return xfer_open_named_socket ( xfer, SOCK_STREAM,
  1472. ( struct sockaddr * ) &peer,
  1473. uri->host, NULL );
  1474. }
  1475. /** TCP URI opener */
  1476. struct uri_opener tcp_uri_opener __uri_opener = {
  1477. .scheme = "tcp",
  1478. .open = tcp_open_uri,
  1479. };