http.c 129 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/********************************************************************
 *                                                                  *
 * THIS FILE IS PART OF THE libopusfile SOFTWARE CODEC SOURCE CODE. *
 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
 *                                                                  *
 * THE libopusfile SOURCE CODE IS (C) COPYRIGHT 2012                *
 * by the Xiph.Org Foundation and contributors http://www.xiph.org/ *
 *                                                                  *
 ********************************************************************/
12
13
14
15
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

16
17
18
19
20
21
#include "internal.h"
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <string.h>

22
/*RFCs referenced in this file:
23
  RFC  761: DOD Standard Transmission Control Protocol
24
25
  RFC 1535: A Security Problem and Proposed Correction With Widely Deployed DNS
   Software
26
27
28
29
30
  RFC 1738: Uniform Resource Locators (URL)
  RFC 1945: Hypertext Transfer Protocol -- HTTP/1.0
  RFC 2068: Hypertext Transfer Protocol -- HTTP/1.1
  RFC 2145: Use and Interpretation of HTTP Version Numbers
  RFC 2246: The TLS Protocol Version 1.0
31
32
  RFC 2459: Internet X.509 Public Key Infrastructure Certificate and
   Certificate Revocation List (CRL) Profile
33
34
35
36
37
38
39
40
  RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1
  RFC 2617: HTTP Authentication: Basic and Digest Access Authentication
  RFC 2817: Upgrading to TLS Within HTTP/1.1
  RFC 2818: HTTP Over TLS
  RFC 3492: Punycode: A Bootstring encoding of Unicode for Internationalized
   Domain Names in Applications (IDNA)
  RFC 3986: Uniform Resource Identifier (URI): Generic Syntax
  RFC 3987: Internationalized Resource Identifiers (IRIs)
41
  RFC 4343: Domain Name System (DNS) Case Insensitivity Clarification
42
43
  RFC 5894: Internationalized Domain Names for Applications (IDNA):
   Background, Explanation, and Rationale
44
45
46
  RFC 6066: Transport Layer Security (TLS) Extensions: Extension Definitions
  RFC 6125: Representation and Verification of Domain-Based Application Service
   Identity within Internet Public Key Infrastructure Using X.509 (PKIX)
47
48
   Certificates in the Context of Transport Layer Security (TLS)
  RFC 6555: Happy Eyeballs: Success with Dual-Stack Hosts*/
49

50
51
52
53
54
55
56
57
58
59
typedef struct OpusParsedURL   OpusParsedURL;
typedef struct OpusStringBuf   OpusStringBuf;
typedef struct OpusHTTPConn    OpusHTTPConn;
typedef struct OpusHTTPStream  OpusHTTPStream;

static char *op_string_range_dup(const char *_start,const char *_end){
  size_t  len;
  char   *ret;
  OP_ASSERT(_start<=_end);
  len=_end-_start;
60
61
  /*This is to help avoid overflow elsewhere, later.*/
  if(OP_UNLIKELY(len>=INT_MAX))return NULL;
62
  ret=(char *)_ogg_malloc(sizeof(*ret)*(len+1));
63
  if(OP_LIKELY(ret!=NULL)){
64
    ret=(char *)memcpy(ret,_start,sizeof(*ret)*(len));
65
66
    ret[len]='\0';
  }
67
68
69
70
71
72
73
  return ret;
}

static char *op_string_dup(const char *_s){
  return op_string_range_dup(_s,_s+strlen(_s));
}

74
75
76
77
78
79
80
81
82
83
84
static char *op_string_tolower(char *_s){
  int i;
  for(i=0;_s[i]!='\0';i++){
    int c;
    c=_s[i];
    if(c>='A'&&c<='Z')c+='a'-'A';
    _s[i]=(char)c;
  }
  return _s;
}

85
86
87
/*URI character classes (from RFC 3986).*/
#define OP_URL_ALPHA \
 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
88
89
#define OP_URL_DIGIT       "0123456789"
#define OP_URL_HEXDIGIT    "0123456789ABCDEFabcdef"
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
/*Not a character class, but the characters allowed in <scheme>.*/
#define OP_URL_SCHEME      OP_URL_ALPHA OP_URL_DIGIT "+-."
#define OP_URL_GEN_DELIMS  "#/:?@[]"
#define OP_URL_SUB_DELIMS  "!$&'()*+,;="
#define OP_URL_RESERVED    OP_URL_GEN_DELIMS OP_URL_SUB_DELIMS
#define OP_URL_UNRESERVED  OP_URL_ALPHA OP_URL_DIGIT "-._~"
/*Not a character class, but the characters allowed in <pct-encoded>.*/
#define OP_URL_PCT_ENCODED "%"
/*Not a character class or production rule, but for convenience.*/
#define OP_URL_PCHAR_BASE \
 OP_URL_UNRESERVED OP_URL_PCT_ENCODED OP_URL_SUB_DELIMS
#define OP_URL_PCHAR       OP_URL_PCHAR_BASE ":@"
/*Not a character class, but the characters allowed in <userinfo> and
   <IP-literal>.*/
#define OP_URL_PCHAR_NA    OP_URL_PCHAR_BASE ":"
/*Not a character class, but the characters allowed in <segment-nz-nc>.*/
#define OP_URL_PCHAR_NC    OP_URL_PCHAR_BASE "@"
/*Not a character clsss, but the characters allowed in <path>.*/
#define OP_URL_PATH        OP_URL_PCHAR "/"
/*Not a character class, but the characters allowed in <query> / <fragment>.*/
#define OP_URL_QUERY_FRAG  OP_URL_PCHAR "/?"

/*Check the <% HEXDIG HEXDIG> escapes of a URL for validity.
  Return: 0 if valid, or a negative value on failure.*/
static int op_validate_url_escapes(const char *_s){
  int i;
  for(i=0;_s[i];i++){
    if(_s[i]=='%'){
      if(OP_UNLIKELY(!isxdigit(_s[i+1]))
       ||OP_UNLIKELY(!isxdigit(_s[i+2]))
       /*RFC 3986 says %00 "should be rejected if the application is not
          expecting to receive raw data within a component."*/
       ||OP_UNLIKELY(_s[i+1]=='0'&&_s[i+2]=='0')){
        return OP_FALSE;
      }
      i+=2;
    }
  }
  return 0;
}

/*Convert a hex digit to its actual value.
  _c: The hex digit to convert.
      Presumed to be valid ('0'...'9', 'A'...'F', or 'a'...'f').
  Return: The value of the digit, in the range [0,15].*/
static int op_hex_value(int _c){
  return _c>='a'?_c-'a'+10:_c>='A'?_c-'A'+10:_c-'0';
}

/*Unescape all the <% HEXDIG HEXDIG> sequences in a string in-place.
  This does no validity checking.*/
static char *op_unescape_url_component(char *_s){
  int i;
  int j;
  for(i=j=0;_s[i];i++,j++){
    if(_s[i]=='%'){
      _s[i]=(char)(op_hex_value(_s[i+1])<<4|op_hex_value(_s[i+2]));
      i+=2;
    }
  }
  return _s;
}

/*Parse a file: URL.
  This code is not meant to be fast: strspn() with large sets is likely to be
   slow, but it is very convenient.
  It is meant to be RFC 1738-compliant (as updated by RFC 3986).*/
static const char *op_parse_file_url(const char *_src){
  const char *scheme_end;
  const char *path;
  const char *path_end;
  scheme_end=_src+strspn(_src,OP_URL_SCHEME);
  if(OP_UNLIKELY(*scheme_end!=':')
   ||scheme_end-_src!=4||op_strncasecmp(_src,"file",4)!=0){
    /*Unsupported protocol.*/
    return NULL;
  }
  /*Make sure all escape sequences are valid to simplify unescaping later.*/
  if(OP_UNLIKELY(op_validate_url_escapes(scheme_end+1)<0))return NULL;
  if(scheme_end[1]=='/'&&scheme_end[2]=='/'){
    const char *host;
    /*file: URLs can have a host!
      Yeah, I was surprised, too, but that's what RFC 1738 says.
      It also says, "The file URL scheme is unusual in that it does not specify
       an Internet protocol or access method for such files; as such, its
       utility in network protocols between hosts is limited," which is a mild
       understatement.*/
    host=scheme_end+3;
    /*The empty host is what we expect.*/
179
    if(OP_LIKELY(*host=='/'))path=host;
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
    else{
      const char *host_end;
      char        host_buf[28];
      /*RFC 1738 says localhost "is interpreted as `the machine from which the
         URL is being interpreted,'" so let's check for it.*/
      host_end=host+strspn(host,OP_URL_PCHAR_BASE);
      /*No <port> allowed.
        This also rejects IP-Literals.*/
      if(*host_end!='/')return NULL;
      /*An escaped "localhost" can take at most 27 characters.*/
      if(OP_UNLIKELY(host_end-host>27))return NULL;
      memcpy(host_buf,host,sizeof(*host_buf)*(host_end-host));
      host_buf[host_end-host]='\0';
      op_unescape_url_component(host_buf);
      op_string_tolower(host_buf);
      /*Some other host: give up.*/
      if(OP_UNLIKELY(strcmp(host_buf,"localhost")!=0))return NULL;
      path=host_end;
    }
199
200
201
202
203
204
205
206
207
208
209
210
211
212
  }
  else path=scheme_end+1;
  path_end=path+strspn(path,OP_URL_PATH);
  /*This will reject a <query> or <fragment> component, too.
    I don't know what to do with queries, but a temporal fragment would at
     least make sense.
    RFC 1738 pretty clearly defines a <searchpart> that's equivalent to the
     RFC 3986 <query> component for other schemes, but not the file: scheme,
     so I'm going to just reject it.*/
  if(*path_end!='\0')return NULL;
  return path;
}

#if defined(OP_ENABLE_HTTP)
213
214
215
# if defined(_WIN32)
#  include <winsock2.h>
#  include <ws2tcpip.h>
216
#  include <openssl/ssl.h>
217
#  include <openssl/asn1.h>
218
219
220
221
222
223
#  include "winerrno.h"

typedef SOCKET op_sock;

#  define OP_INVALID_SOCKET (INVALID_SOCKET)

224
225
226
227
228
229
230
231
232
/*Vista and later support WSAPoll(), but we don't want to rely on that.
  Instead we re-implement it badly using select().
  Unfortunately, they define a conflicting struct pollfd, so we only define our
   own if it looks like that one has not already been defined.*/
#  if !defined(POLLIN)
/*Equivalent to POLLIN.*/
#   define POLLRDNORM (0x0100)
/*Priority band data can be read.*/
#   define POLLRDBAND (0x0200)
233
/*There is data to read.*/
234
#   define POLLIN     (POLLRDNORM|POLLRDBAND)
235
/*There is urgent data to read.*/
236
237
238
#   define POLLPRI    (0x0400)
/*Equivalent to POLLOUT.*/
#   define POLLWRNORM (0x0010)
239
/*Writing now will not block.*/
240
241
242
#   define POLLOUT    (POLLWRNORM)
/*Priority data may be written.*/
#   define POLLWRBAND (0x0020)
243
/*Error condition (output only).*/
244
#   define POLLERR    (0x0001)
245
/*Hang up (output only).*/
246
#   define POLLHUP    (0x0002)
247
/*Invalid request: fd not open (output only).*/
248
#   define POLLNVAL   (0x0004)
249
250
251
252
253
254
255
256
257

struct pollfd{
  /*File descriptor.*/
  op_sock fd;
  /*Requested events.*/
  short   events;
  /*Returned events.*/
  short   revents;
};
258
#  endif
259

260
/*But Winsock never defines nfds_t (it's simply hard-coded to ULONG).*/
261
262
typedef unsigned long nfds_t;

263
/*The usage of FD_SET() below is O(N^2).
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
  This is okay because select() is limited to 64 sockets in Winsock, anyway.
  In practice, we only ever call it with one or two sockets.*/
static int op_poll_win32(struct pollfd *_fds,nfds_t _nfds,int _timeout){
  struct timeval tv;
  fd_set         ifds;
  fd_set         ofds;
  fd_set         efds;
  nfds_t         i;
  int            ret;
  FD_ZERO(&ifds);
  FD_ZERO(&ofds);
  FD_ZERO(&efds);
  for(i=0;i<_nfds;i++){
    _fds[i].revents=0;
    if(_fds[i].events&POLLIN)FD_SET(_fds[i].fd,&ifds);
    if(_fds[i].events&POLLOUT)FD_SET(_fds[i].fd,&ofds);
    FD_SET(_fds[i].fd,&efds);
  }
  if(_timeout>=0){
    tv.tv_sec=_timeout/1000;
    tv.tv_usec=(_timeout%1000)*1000;
  }
  ret=select(-1,&ifds,&ofds,&efds,_timeout<0?NULL:&tv);
  if(ret>0){
    for(i=0;i<_nfds;i++){
      if(FD_ISSET(_fds[i].fd,&ifds))_fds[i].revents|=POLLIN;
      if(FD_ISSET(_fds[i].fd,&ofds))_fds[i].revents|=POLLOUT;
      /*This isn't correct: there are several different things that might have
         happened to a fd in efds, but I don't know a good way to distinguish
         them without more context from the caller.
        It's okay, because we don't actually check any of these bits, we just
         need _some_ bit set.*/
      if(FD_ISSET(_fds[i].fd,&efds))_fds[i].revents|=POLLHUP;
    }
  }
  return ret;
}

/*We define op_errno() to make it clear that it's not an l-value like normal
   errno is.*/
#  define op_errno() (WSAGetLastError()?WSAGetLastError()-WSABASEERR:0)
#  define op_reset_errno() (WSASetLastError(0))

/*The remaining functions don't get an op_ prefix even though they only
   operate on sockets, because we don't use non-socket I/O here, and this
   minimizes the changes needed to deal with Winsock.*/
#  define close(_fd) closesocket(_fd)
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
311
312
313
314
315
316
/*This takes an int for the address length, even though the value is of type
   socklen_t (defined as an unsigned integer type with at least 32 bits).*/
#  define connect(_fd,_addr,_addrlen) \
 (OP_UNLIKELY((_addrlen)>(socklen_t)INT_MAX)? \
  WSASetLastError(WSA_NOT_ENOUGH_MEMORY),-1: \
  connect(_fd,_addr,(int)(_addrlen)))
317
318
319
320
321
322
323
324
325
/*This relies on sizeof(u_long)==sizeof(int), which is always true on both
   Win32 and Win64.*/
#  define ioctl(_fd,_req,_arg) ioctlsocket(_fd,_req,(u_long *)(_arg))
#  define getsockopt(_fd,_level,_name,_val,_len) \
 getsockopt(_fd,_level,_name,(char *)(_val),_len)
#  define setsockopt(_fd,_level,_name,_val,_len) \
 setsockopt(_fd,_level,_name,(const char *)(_val),_len)
#  define poll(_fds,_nfds,_timeout) op_poll_win32(_fds,_nfds,_timeout)

326
327
328
329
#  if defined(_MSC_VER)
typedef ptrdiff_t ssize_t;
#  endif

330
331
332
333
334
/*Load certificates from the built-in certificate store.*/
int SSL_CTX_set_default_verify_paths_win32(SSL_CTX *_ssl_ctx);
#  define SSL_CTX_set_default_verify_paths \
 SSL_CTX_set_default_verify_paths_win32

335
336
337
338
339
340
341
342
343
344
345
346
# else
/*Normal Berkeley sockets.*/
#  include <sys/ioctl.h>
#  include <sys/types.h>
#  include <sys/socket.h>
#  include <arpa/inet.h>
#  include <netinet/in.h>
#  include <netinet/tcp.h>
#  include <fcntl.h>
#  include <netdb.h>
#  include <poll.h>
#  include <unistd.h>
347
#  include <openssl/ssl.h>
348
#  include <openssl/asn1.h>
349
350
351
352
353
354
355
356
357

typedef int op_sock;

#  define OP_INVALID_SOCKET (-1)

#  define op_errno() (errno)
#  define op_reset_errno() (errno=0)

# endif
358
# include <sys/timeb.h>
359
# include <openssl/x509v3.h>
360

361
362
363
364
365
# if (defined(LIBRESSL_VERSION_NUMBER)&&OPENSSL_VERSION_NUMBER==0x20000000L)
#  undef OPENSSL_VERSION_NUMBER
#  define OPENSSL_VERSION_NUMBER 0x1000115fL
# endif

366
367
368
369
370
371
372
373
374
/*The maximum number of simultaneous connections.
  RFC 2616 says this SHOULD NOT be more than 2, but everyone on the modern web
   ignores that (e.g., IE 8 bumped theirs up from 2 to 6, Firefox uses 15).
  If it makes you feel better, we'll only ever actively read from one of these
   at a time.
  The others are kept around mainly to avoid slow-starting a new connection
   when seeking, and time out rapidly.*/
# define OP_NCONNS_MAX (4)

375
376
377
378
379
/*The amount of time before we attempt to re-resolve the host.
  This is 10 minutes, as recommended in RFC 6555 for expiring cached connection
   results for dual-stack hosts.*/
# define OP_RESOLVE_CACHE_TIMEOUT_MS (10*60*(opus_int32)1000)

380
381
382
383
384
385
386
/*The number of redirections at which we give up.
  The value here is the current default in Firefox.
  RFC 2068 mandated a maximum of 5, but RFC 2616 relaxed that to "a client
   SHOULD detect infinite redirection loops."
  Fortunately, 20 is less than infinity.*/
# define OP_REDIRECT_LIMIT (20)

387
388
389
/*The initial size of the buffer used to read a response message (before the
   body).*/
# define OP_RESPONSE_SIZE_MIN (510)
390
391
/*The maximum size of a response message (before the body).
  Responses larger than this will be discarded.
392
393
394
395
  I've seen a real server return 20 kB of data for a 302 Found response.
  Increasing this beyond 32kB will cause problems on platforms with a 16-bit
   int.*/
# define OP_RESPONSE_SIZE_MAX (32766)
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442

/*The number of milliseconds we will allow a connection to sit idle before we
   refuse to resurrect it.
  Apache as of 2.2 has reduced its default timeout to 5 seconds (from 15), so
   that's what we'll use here.*/
# define OP_CONNECTION_IDLE_TIMEOUT_MS (5*1000)

/*The number of milliseconds we will wait to send or receive data before giving
   up.*/
# define OP_POLL_TIMEOUT_MS (30*1000)

/*We will always attempt to read ahead at least this much in preference to
   opening a new connection.*/
# define OP_READAHEAD_THRESH_MIN (32*(opus_int32)1024)

/*The amount of data to request after a seek.
  This is a trade-off between read throughput after a seek vs. the the ability
   to quickly perform another seek with the same connection.*/
# define OP_PIPELINE_CHUNK_SIZE     (32*(opus_int32)1024)
/*Subsequent chunks are requested with larger and larger sizes until they pass
   this threshold, after which we just ask for the rest of the resource.*/
# define OP_PIPELINE_CHUNK_SIZE_MAX (1024*(opus_int32)1024)
/*This is the maximum number of requests we'll make with a single connection.
  Many servers will simply disconnect after we attempt some number of requests,
   possibly without sending a Connection: close header, meaning we won't
   discover it until we try to read beyond the end of the current chunk.
  We can reconnect when that happens, but this is slow.
  Instead, we impose a limit ourselves (set to the default for Apache
   installations and thus likely the most common value in use).*/
# define OP_PIPELINE_MAX_REQUESTS   (100)
/*This should be the number of requests, starting from a chunk size of
   OP_PIPELINE_CHUNK_SIZE and doubling each time, until we exceed
   OP_PIPELINE_CHUNK_SIZE_MAX and just request the rest of the file.
  We won't reuse a connection when seeking unless it has at least this many
   requests left, to reduce the chances we'll have to open a new connection
   while reading forward afterwards.*/
# define OP_PIPELINE_MIN_REQUESTS   (7)

/*Is this an https URL?
  For now we can simply check the last letter of the scheme.*/
# define OP_URL_IS_SSL(_url) ((_url)->scheme[4]=='s')

/*Does this URL use the default port for its scheme?*/
# define OP_URL_IS_DEFAULT_PORT(_url) \
 (!OP_URL_IS_SSL(_url)&&(_url)->port==80 \
 ||OP_URL_IS_SSL(_url)&&(_url)->port==443)

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
struct OpusParsedURL{
  /*Either "http" or "https".*/
  char     *scheme;
  /*The user name from the <userinfo> component, or NULL.*/
  char     *user;
  /*The password from the <userinfo> component, or NULL.*/
  char     *pass;
  /*The <host> component.
    This may not be NULL.*/
  char     *host;
  /*The <path> and <query> components.
    This may not be NULL.*/
  char     *path;
  /*The <port> component.
    This is set to the default port if the URL did not contain one.*/
  unsigned  port;
};

/*Parse a URL.
  This code is not meant to be fast: strspn() with large sets is likely to be
   slow, but it is very convenient.
464
465
466
467
  It is meant to be RFC 3986-compliant.
  We currently do not support IRIs (Internationalized Resource Identifiers,
   RFC 3987).
  Callers should translate them to URIs first.*/
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
static int op_parse_url_impl(OpusParsedURL *_dst,const char *_src){
  const char  *scheme_end;
  const char  *authority;
  const char  *userinfo_end;
  const char  *user;
  const char  *user_end;
  const char  *pass;
  const char  *hostport;
  const char  *hostport_end;
  const char  *host_end;
  const char  *port;
  opus_int32   port_num;
  const char  *port_end;
  const char  *path;
  const char  *path_end;
  const char  *uri_end;
  scheme_end=_src+strspn(_src,OP_URL_SCHEME);
  if(OP_UNLIKELY(*scheme_end!=':')
   ||OP_UNLIKELY(scheme_end-_src<4)||OP_UNLIKELY(scheme_end-_src>5)
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
487
   ||OP_UNLIKELY(op_strncasecmp(_src,"https",(int)(scheme_end-_src))!=0)){
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
    /*Unsupported protocol.*/
    return OP_EIMPL;
  }
  if(OP_UNLIKELY(scheme_end[1]!='/')||OP_UNLIKELY(scheme_end[2]!='/')){
    /*We require an <authority> component.*/
    return OP_EINVAL;
  }
  authority=scheme_end+3;
  /*Make sure all escape sequences are valid to simplify unescaping later.*/
  if(OP_UNLIKELY(op_validate_url_escapes(authority)<0))return OP_EINVAL;
  /*Look for a <userinfo> component.*/
  userinfo_end=authority+strspn(authority,OP_URL_PCHAR_NA);
  if(*userinfo_end=='@'){
    /*Found one.*/
    user=authority;
    /*Look for a password (yes, clear-text passwords are deprecated, I know,
       but what else are people supposed to use? use SSL if you care).*/
    user_end=authority+strspn(authority,OP_URL_PCHAR_BASE);
    if(*user_end==':')pass=user_end+1;
    else pass=NULL;
    hostport=userinfo_end+1;
  }
  else{
511
512
513
    /*We shouldn't have to initialize user_end, but gcc is too dumb to figure
       out that user!=NULL below means we didn't take this else branch.*/
    user=user_end=NULL;
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
    pass=NULL;
    hostport=authority;
  }
  /*Try to figure out where the <host> component ends.*/
  if(hostport[0]=='['){
    hostport++;
    /*We have an <IP-literal>, which can contain colons.*/
    hostport_end=host_end=hostport+strspn(hostport,OP_URL_PCHAR_NA);
    if(OP_UNLIKELY(*hostport_end++!=']'))return OP_EINVAL;
  }
  /*Currently we don't support IDNA (RFC 5894), because I don't want to deal
     with the policy about which domains should not be internationalized to
     avoid confusing similarities.
    Give this API Punycode (RFC 3492) domain names instead.*/
  else hostport_end=host_end=hostport+strspn(hostport,OP_URL_PCHAR_BASE);
  /*TODO: Validate host.*/
  /*Is there a port number?*/
  port_num=-1;
  if(*hostport_end==':'){
    int i;
    port=hostport_end+1;
    port_end=port+strspn(port,OP_URL_DIGIT);
    path=port_end;
    /*Not part of RFC 3986, but require port numbers in the range 0...65535.*/
    if(OP_LIKELY(port_end-port>0)){
      while(*port=='0')port++;
      if(OP_UNLIKELY(port_end-port>5))return OP_EINVAL;
      port_num=0;
      for(i=0;i<port_end-port;i++)port_num=port_num*10+port[i]-'0';
      if(OP_UNLIKELY(port_num>65535))return OP_EINVAL;
    }
  }
  else path=hostport_end;
  path_end=path+strspn(path,OP_URL_PATH);
  /*If the path is not empty, it must begin with a '/'.*/
  if(OP_LIKELY(path_end>path)&&OP_UNLIKELY(path[0]!='/'))return OP_EINVAL;
  /*Consume the <query> component, if any (right now we don't split this out
     from the <path> component).*/
  if(*path_end=='?')path_end=path_end+strspn(path_end,OP_URL_QUERY_FRAG);
  /*Discard the <fragment> component, if any.
    This doesn't get sent to the server.
    Some day we should add support for Media Fragment URIs
     <http://www.w3.org/TR/media-frags/>.*/
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
557
  if(*path_end=='#')uri_end=path_end+1+strspn(path_end+1,OP_URL_QUERY_FRAG);
558
559
  else uri_end=path_end;
  /*If there's anything left, this was not a valid URL.*/
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
560
  if(OP_UNLIKELY(*uri_end!='\0'))return OP_EINVAL;
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
  _dst->scheme=op_string_range_dup(_src,scheme_end);
  if(OP_UNLIKELY(_dst->scheme==NULL))return OP_EFAULT;
  op_string_tolower(_dst->scheme);
  if(user!=NULL){
    _dst->user=op_string_range_dup(user,user_end);
    if(OP_UNLIKELY(_dst->user==NULL))return OP_EFAULT;
    op_unescape_url_component(_dst->user);
    /*Unescaping might have created a ':' in the username.
      That's not allowed by RFC 2617's Basic Authentication Scheme.*/
    if(OP_UNLIKELY(strchr(_dst->user,':')!=NULL))return OP_EINVAL;
  }
  else _dst->user=NULL;
  if(pass!=NULL){
    _dst->pass=op_string_range_dup(pass,userinfo_end);
    if(OP_UNLIKELY(_dst->pass==NULL))return OP_EFAULT;
    op_unescape_url_component(_dst->pass);
  }
  else _dst->pass=NULL;
  _dst->host=op_string_range_dup(hostport,host_end);
  if(OP_UNLIKELY(_dst->host==NULL))return OP_EFAULT;
  if(port_num<0){
    if(_src[4]=='s')port_num=443;
    else port_num=80;
  }
  _dst->port=(unsigned)port_num;
  /*RFC 2616 says an empty <abs-path> component is equivalent to "/", and we
     MUST use the latter in the Request-URI.
    Reserve space for the slash here.*/
  if(path==path_end||path[0]=='?')path--;
  _dst->path=op_string_range_dup(path,path_end);
  if(OP_UNLIKELY(_dst->path==NULL))return OP_EFAULT;
  /*And force-set it here.*/
  _dst->path[0]='/';
  return 0;
}

static void op_parsed_url_init(OpusParsedURL *_url){
  memset(_url,0,sizeof(*_url));
}

static void op_parsed_url_clear(OpusParsedURL *_url){
  _ogg_free(_url->scheme);
  _ogg_free(_url->user);
  _ogg_free(_url->pass);
  _ogg_free(_url->host);
  _ogg_free(_url->path);
}

static int op_parse_url(OpusParsedURL *_dst,const char *_src){
  OpusParsedURL url;
  int           ret;
  op_parsed_url_init(&url);
  ret=op_parse_url_impl(&url,_src);
  if(OP_UNLIKELY(ret<0))op_parsed_url_clear(&url);
  else *_dst=*&url;
  return ret;
}

/*A buffer to hold growing strings.
  The main purpose of this is to consolidate allocation checks and simplify
   cleanup on a failed allocation.*/
struct OpusStringBuf{
  char *buf;
  int   nbuf;
  int   cbuf;
};

static void op_sb_init(OpusStringBuf *_sb){
  _sb->buf=NULL;
  _sb->nbuf=0;
  _sb->cbuf=0;
}

static void op_sb_clear(OpusStringBuf *_sb){
  _ogg_free(_sb->buf);
}

638
639
/*Make sure we have room for at least _capacity characters (plus 1 more for the
   terminating NUL).*/
640
641
642
643
644
645
static int op_sb_ensure_capacity(OpusStringBuf *_sb,int _capacity){
  char *buf;
  int   cbuf;
  buf=_sb->buf;
  cbuf=_sb->cbuf;
  if(_capacity>=cbuf-1){
646
    if(OP_UNLIKELY(cbuf>INT_MAX-1>>1))return OP_EFAULT;
647
648
649
650
651
652
653
654
655
656
    if(OP_UNLIKELY(_capacity>=INT_MAX-1))return OP_EFAULT;
    cbuf=OP_MAX(2*cbuf+1,_capacity+1);
    buf=_ogg_realloc(buf,sizeof(*buf)*cbuf);
    if(OP_UNLIKELY(buf==NULL))return OP_EFAULT;
    _sb->buf=buf;
    _sb->cbuf=cbuf;
  }
  return 0;
}

657
658
/*Increase the capacity of the buffer, but not to more than _max_size
   characters (plus 1 more for the terminating NUL).*/
659
660
661
662
663
664
665
666
667
668
669
670
671
672
static int op_sb_grow(OpusStringBuf *_sb,int _max_size){
  char *buf;
  int   cbuf;
  buf=_sb->buf;
  cbuf=_sb->cbuf;
  OP_ASSERT(_max_size<=INT_MAX-1);
  cbuf=cbuf<=_max_size-1>>1?2*cbuf+1:_max_size+1;
  buf=_ogg_realloc(buf,sizeof(*buf)*cbuf);
  if(OP_UNLIKELY(buf==NULL))return OP_EFAULT;
  _sb->buf=buf;
  _sb->cbuf=cbuf;
  return 0;
}

673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
static int op_sb_append(OpusStringBuf *_sb,const char *_s,int _len){
  char *buf;
  int   nbuf;
  int   ret;
  nbuf=_sb->nbuf;
  if(OP_UNLIKELY(nbuf>INT_MAX-_len))return OP_EFAULT;
  ret=op_sb_ensure_capacity(_sb,nbuf+_len);
  if(OP_UNLIKELY(ret<0))return ret;
  buf=_sb->buf;
  memcpy(buf+nbuf,_s,sizeof(*buf)*_len);
  nbuf+=_len;
  buf[nbuf]='\0';
  _sb->nbuf=nbuf;
  return 0;
}

static int op_sb_append_string(OpusStringBuf *_sb,const char *_s){
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
690
691
692
693
  size_t len;
  len=strlen(_s);
  if(OP_UNLIKELY(len>(size_t)INT_MAX))return OP_EFAULT;
  return op_sb_append(_sb,_s,(int)len);
694
695
}

696
697
698
699
700
701
702
static int op_sb_append_port(OpusStringBuf *_sb,unsigned _port){
  char port_buf[7];
  OP_ASSERT(_port<=65535U);
  sprintf(port_buf,":%u",_port);
  return op_sb_append_string(_sb,port_buf);
}

703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
static int op_sb_append_nonnegative_int64(OpusStringBuf *_sb,opus_int64 _i){
  char digit;
  int  nbuf_start;
  int  ret;
  OP_ASSERT(_i>=0);
  nbuf_start=_sb->nbuf;
  ret=0;
  do{
    digit='0'+_i%10;
    ret|=op_sb_append(_sb,&digit,1);
    _i/=10;
  }
  while(_i>0);
  if(OP_LIKELY(ret>=0)){
    char *buf;
    int   nbuf_end;
    buf=_sb->buf;
    nbuf_end=_sb->nbuf-1;
    /*We've added the digits backwards.
      Reverse them.*/
    while(nbuf_start<nbuf_end){
      digit=buf[nbuf_start];
      buf[nbuf_start]=buf[nbuf_end];
      buf[nbuf_end]=digit;
      nbuf_start++;
      nbuf_end--;
    }
  }
  return ret;
}

734
735
736
737
738
739
static struct addrinfo *op_resolve(const char *_host,unsigned _port){
  struct addrinfo *addrs;
  struct addrinfo  hints;
  char             service[6];
  memset(&hints,0,sizeof(hints));
  hints.ai_socktype=SOCK_STREAM;
740
#if defined(AI_NUMERICSERV)
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
741
  hints.ai_flags=AI_NUMERICSERV;
742
#endif
743
744
745
746
747
748
  OP_ASSERT(_port<=65535U);
  sprintf(service,"%u",_port);
  if(OP_LIKELY(!getaddrinfo(_host,service,&hints,&addrs)))return addrs;
  return NULL;
}

749
750
static int op_sock_set_nonblocking(op_sock _fd,int _nonblocking){
#if !defined(_WIN32)
751
752
753
754
755
756
  int flags;
  flags=fcntl(_fd,F_GETFL);
  if(OP_UNLIKELY(flags<0))return flags;
  if(_nonblocking)flags|=O_NONBLOCK;
  else flags&=~O_NONBLOCK;
  return fcntl(_fd,F_SETFL,flags);
757
#else
758
  return ioctl(_fd,FIONBIO,&_nonblocking);
759
#endif
760
761
762
763
764
765
}

/*Disable/enable write coalescing if we can.
  We always send whole requests at once and always parse the response headers
   before sending another one, so normally write coalescing just causes added
   delay.*/
766
static void op_sock_set_tcp_nodelay(op_sock _fd,int _nodelay){
767
768
769
770
771
772
773
774
# if defined(TCP_NODELAY)&&(defined(IPPROTO_TCP)||defined(SOL_TCP))
#  if defined(IPPROTO_TCP)
#   define OP_SO_LEVEL IPPROTO_TCP
#  else
#   define OP_SO_LEVEL SOL_TCP
#  endif
  /*It doesn't really matter if this call fails, but it would be interesting
     to hit a case where it does.*/
775
  OP_ALWAYS_TRUE(!setsockopt(_fd,OP_SO_LEVEL,TCP_NODELAY,
776
   &_nodelay,sizeof(_nodelay)));
777
778
779
# endif
}

780
#if defined(_WIN32)
781
static void op_init_winsock(){
782
  static LONG    count;
783
  static WSADATA wsadata;
784
  if(InterlockedIncrement(&count)==1)WSAStartup(0x0202,&wsadata);
785
786
787
}
#endif

788
789
/*A single physical connection to an HTTP server.
  We may have several of these open at once.*/
790
791
792
struct OpusHTTPConn{
  /*The current position indicator for this connection.*/
  opus_int64    pos;
793
794
795
796
797
798
799
800
801
  /*The position where the current request will end, or -1 if we're reading
     until EOF (an unseekable stream or the initial HTTP/1.0 request).*/
  opus_int64    end_pos;
  /*The position where next request we've sent will start, or -1 if we haven't
     sent the next request yet.*/
  opus_int64    next_pos;
  /*The end of the next request or -1 if we requested the rest of the resource.
    This is only set to a meaningful value if next_pos is not -1.*/
  opus_int64    next_end;
802
  /*The SSL connection, if this is https.*/
803
  SSL          *ssl_conn;
804
  /*The next connection in either the LRU or free list.*/
805
  OpusHTTPConn *next;
806
807
808
809
810
811
812
  /*The last time we blocked for reading from this connection.*/
  struct timeb  read_time;
  /*The number of bytes we've read since the last time we blocked.*/
  opus_int64    read_bytes;
  /*The estimated throughput of this connection, in bytes/s.*/
  opus_int64    read_rate;
  /*The socket we're reading from.*/
813
  op_sock       fd;
814
815
816
817
  /*The number of remaining requests we are allowed on this connection.*/
  int           nrequests_left;
  /*The chunk size to use for pipelining requests.*/
  opus_int32    chunk_size;
818
819
820
};

static void op_http_conn_init(OpusHTTPConn *_conn){
821
  _conn->next_pos=-1;
822
823
  _conn->ssl_conn=NULL;
  _conn->next=NULL;
824
  _conn->fd=OP_INVALID_SOCKET;
825
826
827
828
829
}

static void op_http_conn_clear(OpusHTTPConn *_conn){
  if(_conn->ssl_conn!=NULL)SSL_free(_conn->ssl_conn);
  /*SSL frees the BIO for us.*/
830
  if(_conn->fd!=OP_INVALID_SOCKET)close(_conn->fd);
831
832
}

833
/*The global stream state.*/
834
835
836
837
838
struct OpusHTTPStream{
  /*The list of connections.*/
  OpusHTTPConn     conns[OP_NCONNS_MAX];
  /*The context object used as a framework for TLS/SSL functions.*/
  SSL_CTX         *ssl_ctx;
839
840
  /*The cached session to reuse for future connections.*/
  SSL_SESSION     *ssl_session;
841
842
  /*The LRU list (ordered from MRU to LRU) of currently connected
     connections.*/
843
844
845
846
847
848
849
850
851
852
853
854
855
  OpusHTTPConn    *lru_head;
  /*The free list.*/
  OpusHTTPConn    *free_head;
  /*The URL to connect to.*/
  OpusParsedURL    url;
  /*Information about the address we connected to.*/
  struct addrinfo  addr_info;
  /*The address we connected to.*/
  union{
    struct sockaddr     s;
    struct sockaddr_in  v4;
    struct sockaddr_in6 v6;
  }                addr;
856
857
  /*The last time we re-resolved the host.*/
  struct timeb     resolve_time;
858
859
  /*A buffer used to build HTTP requests.*/
  OpusStringBuf    request;
860
861
  /*A buffer used to build proxy CONNECT requests.*/
  OpusStringBuf    proxy_connect;
862
863
  /*A buffer used to receive the response headers.*/
  OpusStringBuf    response;
864
  /*The Content-Length, if specified, or -1 otherwise.
865
    This will always be specified for seekable streams.*/
866
867
868
  opus_int64       content_length;
  /*The position indicator used when no connection is active.*/
  opus_int64       pos;
869
870
871
872
  /*The host we actually connected to.*/
  char            *connect_host;
  /*The port we actually connected to.*/
  unsigned         connect_port;
873
874
875
876
877
878
879
  /*The connection we're currently reading from.
    This can be -1 if no connection is active.*/
  int              cur_conni;
  /*Whether or not the server supports range requests.*/
  int              seekable;
  /*Whether or not the server supports HTTP/1.1 with persistent connections.*/
  int              pipeline;
880
881
  /*Whether or not we should skip certificate checks.*/
  int              skip_certificate_check;
882
883
884
885
886
887
  /*The offset of the tail of the request.
    Only the offset in the Range: header appears after this, allowing us to
     quickly edit the request to ask for a new range.*/
  int              request_tail;
  /*The estimated time required to open a new connection, in milliseconds.*/
  opus_int32       connect_rate;
888
889
890
891
};

static void op_http_stream_init(OpusHTTPStream *_stream){
  OpusHTTPConn **pnext;
892
  int            ci;
893
894
895
896
897
898
899
  pnext=&_stream->free_head;
  for(ci=0;ci<OP_NCONNS_MAX;ci++){
    op_http_conn_init(_stream->conns+ci);
    *pnext=_stream->conns+ci;
    pnext=&_stream->conns[ci].next;
  }
  _stream->ssl_ctx=NULL;
900
  _stream->ssl_session=NULL;
901
902
  _stream->lru_head=NULL;
  op_parsed_url_init(&_stream->url);
903
  op_sb_init(&_stream->request);
904
  op_sb_init(&_stream->proxy_connect);
905
  op_sb_init(&_stream->response);
906
  _stream->connect_host=NULL;
907
908
909
  _stream->seekable=0;
}

910
911
912
/*Close the connection and move it to the free list.
  _stream:     The stream containing the free list.
  _conn:       The connection to close.
913
  _pnext:      The linked-list pointer currently pointing to this connection.
914
915
916
917
918
919
920
921
922
923
  _gracefully: Whether or not to shut down cleanly.*/
static void op_http_conn_close(OpusHTTPStream *_stream,OpusHTTPConn *_conn,
 OpusHTTPConn **_pnext,int _gracefully){
  /*If we don't shut down gracefully, the server MUST NOT re-use our session
     according to RFC 2246, because it can't tell the difference between an
     abrupt close and a truncation attack.
    So we shut down gracefully if we can.
    However, we will not wait if this would block (it's not worth the savings
     from session resumption to do so).
    Clients (that's us) MAY resume a TLS session that ended with an incomplete
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
924
925
     close, according to RFC 2818, so there's no reason to make sure the server
     shut things down gracefully.*/
926
  if(_gracefully&&_conn->ssl_conn!=NULL)SSL_shutdown(_conn->ssl_conn);
927
  op_http_conn_clear(_conn);
928
  _conn->next_pos=-1;
929
  _conn->ssl_conn=NULL;
930
  _conn->fd=OP_INVALID_SOCKET;
931
932
  OP_ASSERT(*_pnext==_conn);
  *_pnext=_conn->next;
933
934
935
936
937
  _conn->next=_stream->free_head;
  _stream->free_head=_conn;
}

static void op_http_stream_clear(OpusHTTPStream *_stream){
938
939
940
  while(_stream->lru_head!=NULL){
    op_http_conn_close(_stream,_stream->lru_head,&_stream->lru_head,0);
  }
941
  if(_stream->ssl_session!=NULL)SSL_SESSION_free(_stream->ssl_session);
942
  if(_stream->ssl_ctx!=NULL)SSL_CTX_free(_stream->ssl_ctx);
943
  op_sb_clear(&_stream->response);
944
  op_sb_clear(&_stream->proxy_connect);
945
  op_sb_clear(&_stream->request);
946
  if(_stream->connect_host!=_stream->url.host)_ogg_free(_stream->connect_host);
947
948
949
  op_parsed_url_clear(&_stream->url);
}

950
static int op_http_conn_write_fully(OpusHTTPConn *_conn,
951
 const char *_buf,int _buf_size){
952
953
954
955
  struct pollfd  fd;
  SSL           *ssl_conn;
  fd.fd=_conn->fd;
  ssl_conn=_conn->ssl_conn;
956
  while(_buf_size>0){
957
958
959
    int err;
    if(ssl_conn!=NULL){
      int ret;
960
      ret=SSL_write(ssl_conn,_buf,_buf_size);
961
962
963
      if(ret>0){
        /*Wrote some data.*/
        _buf+=ret;
964
        _buf_size-=ret;
965
966
967
968
969
970
971
972
973
974
975
976
        continue;
      }
      /*Connection closed.*/
      else if(ret==0)return OP_FALSE;
      err=SSL_get_error(ssl_conn,ret);
      /*Yes, renegotiations can cause SSL_write() to block for reading.*/
      if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
      else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
      else return OP_FALSE;
    }
    else{
      ssize_t ret;
977
      op_reset_errno();
978
      ret=send(fd.fd,_buf,_buf_size,0);
979
980
      if(ret>0){
        _buf+=ret;
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
981
982
        OP_ASSERT(ret<=_buf_size);
        _buf_size-=(int)ret;
983
984
        continue;
      }
985
      err=op_errno();
986
987
988
989
990
991
      if(err!=EAGAIN&&err!=EWOULDBLOCK)return OP_FALSE;
      fd.events=POLLOUT;
    }
    if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return OP_FALSE;
  }
  return 0;
992
993
}

994
995
996
static int op_http_conn_estimate_available(OpusHTTPConn *_conn){
  int available;
  int ret;
997
  ret=ioctl(_conn->fd,FIONREAD,&available);
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
  if(ret<0)available=0;
  /*This requires the SSL read_ahead flag to be unset to work.
    We ignore partial records as well as the protocol overhead for any pending
     bytes.
    This means we might return somewhat less than can truly be read without
     blocking (if there's a partial record).
    This is okay, because we're using this value to estimate network transfer
     time, and we _have_ already received those bytes.
    We also might return slightly more (due to protocol overhead), but that's
     small enough that it probably doesn't matter.*/
  if(_conn->ssl_conn!=NULL)available+=SSL_pending(_conn->ssl_conn);
  return available;
1010
1011
}

1012
1013
1014
static opus_int32 op_time_diff_ms(const struct timeb *_end,
 const struct timeb *_start){
  opus_int64 dtime;
1015
  dtime=_end->time-(opus_int64)_start->time;
1016
1017
  OP_ASSERT(_end->millitm<1000);
  OP_ASSERT(_start->millitm<1000);
1018
1019
  if(OP_UNLIKELY(dtime>(OP_INT32_MAX-1000)/1000))return OP_INT32_MAX;
  if(OP_UNLIKELY(dtime<(OP_INT32_MIN+1000)/1000))return OP_INT32_MIN;
1020
  return (opus_int32)dtime*1000+_end->millitm-_start->millitm;
1021
1022
}

1023
1024
1025
1026
1027
1028
1029
1030
/*Update the read rate estimate for this connection.*/
static void op_http_conn_read_rate_update(OpusHTTPConn *_conn){
  struct timeb read_time;
  opus_int32   read_delta_ms;
  opus_int64   read_delta_bytes;
  opus_int64   read_rate;
  read_delta_bytes=_conn->read_bytes;
  if(read_delta_bytes<=0)return;
1031
  ftime(&read_time);
1032
1033
1034
1035
1036
1037
1038
1039
  read_delta_ms=op_time_diff_ms(&read_time,&_conn->read_time);
  read_rate=_conn->read_rate;
  read_delta_ms=OP_MAX(read_delta_ms,1);
  read_rate+=read_delta_bytes*1000/read_delta_ms-read_rate+4>>3;
  *&_conn->read_time=*&read_time;
  _conn->read_bytes=0;
  _conn->read_rate=read_rate;
}
1040

1041
1042
/*Tries to read from the given connection.
  [out] _buf: Returns the data read.
1043
1044
1045
1046
1047
1048
  _buf_size:  The size of the buffer.
  _blocking:  Whether or not to block until some data is retrieved.
  Return: A positive number of bytes read on success.
          0:        The read would block, or the connection was closed.
          OP_EREAD: There was a fatal read error.*/
static int op_http_conn_read(OpusHTTPConn *_conn,
1049
 char *_buf,int _buf_size,int _blocking){
1050
1051
1052
1053
  struct pollfd  fd;
  SSL           *ssl_conn;
  int            nread;
  int            nread_unblocked;
1054
1055
1056
  fd.fd=_conn->fd;
  ssl_conn=_conn->ssl_conn;
  nread=nread_unblocked=0;
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1057
1058
1059
  /*RFC 2818 says "client implementations MUST treat any premature closes as
     errors and the data received as potentially truncated," so we make very
     sure to report read errors upwards.*/
1060
  do{
1061
    int err;
1062
1063
    if(ssl_conn!=NULL){
      int ret;
1064
1065
      ret=SSL_read(ssl_conn,_buf+nread,_buf_size-nread);
      OP_ASSERT(ret<=_buf_size-nread);
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
      if(ret>0){
        /*Read some data.
          Keep going to see if there's more.*/
        nread+=ret;
        nread_unblocked+=ret;
        continue;
      }
      /*If we already read some data, return it right now.*/
      if(nread>0)break;
      err=SSL_get_error(ssl_conn,ret);
1076
1077
1078
1079
1080
1081
1082
      if(ret==0){
        /*Connection close.
          Check for a clean shutdown to prevent truncation attacks.
          This check always succeeds for SSLv2, as it has no "close notify"
           message and thus can't verify an orderly shutdown.*/
        return err==SSL_ERROR_ZERO_RETURN?0:OP_EREAD;
      }
1083
1084
1085
      if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
      /*Yes, renegotiations can cause SSL_read() to block for writing.*/
      else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
1086
1087
      /*Some other error.*/
      else return OP_EREAD;
1088
1089
1090
    }
    else{
      ssize_t ret;
1091
      op_reset_errno();
1092
      ret=recv(fd.fd,_buf+nread,_buf_size-nread,0);
1093
      OP_ASSERT(ret<=_buf_size-nread);
1094
1095
1096
      if(ret>0){
        /*Read some data.
          Keep going to see if there's more.*/
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1097
1098
1099
        OP_ASSERT(ret<=_buf_size-nread);
        nread+=(int)ret;
        nread_unblocked+=(int)ret;
1100
1101
        continue;
      }
1102
1103
1104
      /*If we already read some data or the connection was closed, return
         right now.*/
      if(ret==0||nread>0)break;
1105
      err=op_errno();
1106
      if(err!=EAGAIN&&err!=EWOULDBLOCK)return OP_EREAD;
1107
1108
1109
1110
1111
1112
1113
      fd.events=POLLIN;
    }
    _conn->read_bytes+=nread_unblocked;
    op_http_conn_read_rate_update(_conn);
    nread_unblocked=0;
    if(!_blocking)break;
    /*Need to wait to get any data at all.*/
1114
    if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return OP_EREAD;
1115
  }
1116
  while(nread<_buf_size);
1117
  _conn->read_bytes+=nread_unblocked;
1118
1119
1120
  return nread;
}

1121
1122
/*Tries to look at the pending data for a connection without consuming it.
  [out] _buf: Returns the data at which we're peeking.
1123
  _buf_size:  The size of the buffer.*/
1124
static int op_http_conn_peek(OpusHTTPConn *_conn,char *_buf,int _buf_size){
1125
1126
1127
1128
1129
1130
1131
1132
  struct pollfd   fd;
  SSL            *ssl_conn;
  int             ret;
  fd.fd=_conn->fd;
  ssl_conn=_conn->ssl_conn;
  for(;;){
    int err;
    if(ssl_conn!=NULL){
1133
      ret=SSL_peek(ssl_conn,_buf,_buf_size);
1134
1135
1136
1137
1138
1139
1140
1141
1142
      /*Either saw some data or the connection was closed.*/
      if(ret>=0)return ret;
      err=SSL_get_error(ssl_conn,ret);
      if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
      /*Yes, renegotiations can cause SSL_peek() to block for writing.*/
      else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
      else return 0;
    }
    else{
1143
      op_reset_errno();
1144
      ret=(int)recv(fd.fd,_buf,_buf_size,MSG_PEEK);
1145
1146
      /*Either saw some data or the connection was closed.*/
      if(ret>=0)return ret;
1147
      err=op_errno();
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
      if(err!=EAGAIN&&err!=EWOULDBLOCK)return 0;
      fd.events=POLLIN;
    }
    /*Need to wait to get any data at all.*/
    if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return 0;
  }
}

/*When parsing response headers, RFC 2616 mandates that all lines end in CR LF.
  However, even in the year 2012, I have seen broken servers use just a LF.
  This is the evil that Postel's advice from RFC 761 breeds.*/

/*Reads the entirety of a response to an HTTP request into the response buffer.
1161
1162
1163
1164
  Actual parsing and validation is done later.
  Return: The number of bytes in the response on success, OP_EREAD if the
           connection was closed before reading any data, or another negative
           value on any other error.*/
1165
static int op_http_conn_read_response(OpusHTTPConn *_conn,
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
 OpusStringBuf *_response){
  int ret;
  _response->nbuf=0;
  ret=op_sb_ensure_capacity(_response,OP_RESPONSE_SIZE_MIN);
  if(OP_UNLIKELY(ret<0))return ret;
  for(;;){
    char *buf;
    int   size;
    int   capacity;
    int   read_limit;
    int   terminated;
    size=_response->nbuf;
    capacity=_response->cbuf-1;
    if(OP_UNLIKELY(size>=capacity)){
      ret=op_sb_grow(_response,OP_RESPONSE_SIZE_MAX);
      if(OP_UNLIKELY(ret<0))return ret;
      capacity=_response->cbuf-1;
      /*The response was too large.
        This prevents a bad server from running us out of memory.*/
      if(OP_UNLIKELY(size>=capacity))return OP_EIMPL;
    }
    buf=_response->buf;
    ret=op_http_conn_peek(_conn,buf+size,capacity-size);
    if(OP_UNLIKELY(ret<=0))return size<=0?OP_EREAD:OP_FALSE;
1190
1191
    /*We read some data.*/
    /*Make sure the starting characters are "HTTP".
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1192
1193
      Otherwise we could wind up waiting for a response from something that is
       not an HTTP server until we time out.*/
1194
    if(size<4&&op_strncasecmp(buf,"HTTP",OP_MIN(size+ret,4))!=0){
1195
1196
      return OP_FALSE;
    }
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
    /*How far can we read without passing the "\r\n\r\n" terminator?*/
    buf[size+ret]='\0';
    terminated=0;
    for(read_limit=OP_MAX(size-3,0);read_limit<size+ret;read_limit++){
      /*We don't look for the leading '\r' thanks to broken servers.*/
      if(buf[read_limit]=='\n'){
        if(buf[read_limit+1]=='\r'&&OP_LIKELY(buf[read_limit+2]=='\n')){
          terminated=3;
          break;
        }
        /*This case is for broken servers.*/
        else if(OP_UNLIKELY(buf[read_limit+1]=='\n')){
          terminated=2;
          break;
        }
1212
1213
      }
    }
1214
1215
1216
1217
    read_limit+=terminated;
    OP_ASSERT(size<=read_limit);
    OP_ASSERT(read_limit<=size+ret);
    /*Actually consume that data.*/
1218
    ret=op_http_conn_read(_conn,buf+size,read_limit-size,1);
1219
1220
1221
1222
1223
1224
    if(OP_UNLIKELY(ret<=0))return OP_FALSE;
    size+=ret;
    buf[size]='\0';
    _response->nbuf=size;
    /*We found the terminator and read all the data up to and including it.*/
    if(terminated&&OP_LIKELY(size>=read_limit))return size;
1225
1226
1227
1228
  }
  return OP_EIMPL;
}

Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1229
# define OP_HTTP_DIGIT "0123456789"
1230
1231

/*The Reason-Phrase is not allowed to contain control characters, except
1232
   horizontal tab (HT: \011).*/
1233
1234
1235
1236
1237
1238
1239
1240
# define OP_HTTP_CREASON_PHRASE \
 "\001\002\003\004\005\006\007\010\012\013\014\015\016\017\020\021" \
 "\022\023\024\025\026\027\030\031\032\033\034\035\036\037\177"

# define OP_HTTP_CTLS \
 "\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020" \
 "\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\177"

1241
1242
/*This also includes '\t', but we get that from OP_HTTP_CTLS.*/
# define OP_HTTP_SEPARATORS " \"(),/:;<=>?@[\\]{}"
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252

/*TEXT can also include LWS, but that has structure, so we parse it
   separately.*/
# define OP_HTTP_CTOKEN OP_HTTP_CTLS OP_HTTP_SEPARATORS

/*Return: The amount of linear white space (LWS) at the start of _s.*/
static int op_http_lwsspn(const char *_s){
  int i;
  for(i=0;;){
    if(_s[0]=='\r'&&_s[1]=='\n'&&(_s[2]=='\t'||_s[2]==' '))i+=3;
1253
1254
    /*This case is for broken servers.*/
    else if(_s[0]=='\n'&&(_s[1]=='\t'||_s[1]==' '))i+=2;
1255
1256
1257
1258
1259
    else if(_s[i]=='\t'||_s[i]==' ')i++;
    else return i;
  }
}

1260
1261
static char *op_http_parse_status_line(int *_v1_1_compat,
 char **_status_code,char *_response){
1262
1263
  char   *next;
  char   *status_code;
1264
  int     v1_1_compat;
1265
  size_t  d;
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1266
1267
  /*RFC 2616 Section 6.1 does not say if the tokens in the Status-Line can be
     separated by optional LWS, but since it specifically calls out where
1268
     spaces are to be placed and that CR and LF are not allowed except at the
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1269
     end, we are assuming extra LWS is not allowed.*/
1270
1271
1272
1273
1274
  /*We already validated that this starts with "HTTP"*/
  OP_ASSERT(op_strncasecmp(_response,"HTTP",4)==0);
  next=_response+4;
  if(OP_UNLIKELY(*next++!='/'))return NULL;
  d=strspn(next,OP_HTTP_DIGIT);
1275
1276
1277
1278
1279
1280
1281
1282
  /*"Leading zeros MUST be ignored by recipients."*/
  while(*next=='0'){
    next++;
    OP_ASSERT(d>0);
    d--;
  }
  /*We only support version 1.x*/
  if(OP_UNLIKELY(d!=1)||OP_UNLIKELY(*next++!='1'))return NULL;
1283
1284
1285
  if(OP_UNLIKELY(*next++!='.'))return NULL;
  d=strspn(next,OP_HTTP_DIGIT);
  if(OP_UNLIKELY(d<=0))return NULL;
1286
1287
1288
1289
1290
1291
1292
  /*"Leading zeros MUST be ignored by recipients."*/
  while(*next=='0'){
    next++;
    OP_ASSERT(d>0);
    d--;
  }
  /*We don't need to parse the version number.
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
1293
    Any non-zero digit means it's at least 1.*/
1294
  v1_1_compat=d>0;
1295
1296
1297
1298
1299
1300
1301
1302
1303
  next+=d;
  if(OP_UNLIKELY(*next++!=' '))return NULL;
  status_code=next;
  d=strspn(next,OP_HTTP_DIGIT);
  if(OP_UNLIKELY(d!=3))return NULL;
  next+=d;
  /*The Reason-Phrase can be empty, but the space must be here.*/
  if(OP_UNLIKELY(*next++!=' '))return NULL;
  next+=strcspn(next,OP_HTTP_CREASON_PHRASE);
1304
1305
  /*We are not mandating this be present thanks to broken servers.*/
  if(OP_LIKELY(*next=='\r'))next++;
1306
  if(OP_UNLIKELY(*next++!='\n'))return NULL;
1307
  if(_v1_1_compat!=NULL)*_v1_1_compat=v1_1_compat;
1308
1309
1310
1311
  *_status_code=status_code;
  return next;
}

1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
/*Get the next response header.
  [out] _header: The header token, NUL-terminated, with leading and trailing
                  whitespace stripped, and converted to lower case (to simplify
                  case-insensitive comparisons), or NULL if there are no more
                  response headers.
  [out] _cdr:    The remaining contents of the header, excluding the initial
                  colon (':') and the terminating CRLF ("\r\n"),
                  NUL-terminated, and with leading and trailing whitespace
                  stripped, or NULL if there are no more response headers.
  [inout] _s:    On input, this points to the start of the current line of the
                  response headers.
                 On output, it points to the start of the first line following
                  this header, or NULL if there are no more response headers.
  Return: 0 on success, or a negative value on failure.*/
1326
1327
1328
1329
1330
1331
1332
1333
static int op_http_get_next_header(char **_header,char **_cdr,char **_s){
  char   *header;
  char   *header_end;
  char   *cdr;
  char   *cdr_end;
  char   *next;
  size_t  d;
  next=*_s;
1334
1335
  /*The second case is for broken servers.*/
  if(next[0]=='\r'&&next[1]=='\n'||OP_UNLIKELY(next[0]=='\n')){
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
    /*No more headers.*/
    *_header=NULL;
    *_cdr=NULL;
    *_s=NULL;
    return 0;
  }
  header=next+op_http_lwsspn(next);
  d=strcspn(header,OP_HTTP_CTOKEN);
  if(OP_UNLIKELY(d<=0))return OP_FALSE;
  header_end=header+d;
  next=header_end+op_http_lwsspn(header_end);
  if(OP_UNLIKELY(*next++!=':'))return OP_FALSE;
  next+=op_http_lwsspn(next);
  cdr=next;
  do{
    cdr_end=next+strcspn(next,OP_HTTP_CTLS);
    next=cdr_end+op_http_lwsspn(cdr_end);
  }
  while(next>cdr_end);
1355
1356
  /*We are not mandating this be present thanks to broken servers.*/
  if(OP_LIKELY(*next=='\r'))next++;
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
  if(OP_UNLIKELY(*next++!='\n'))return OP_FALSE;
  *header_end='\0';
  *cdr_end='\0';
  /*Field names are case-insensitive.*/
  op_string_tolower(header);
  *_header=header;
  *_cdr=cdr;
  *_s=next;
  return 0;
}

static opus_int64 op_http_parse_nonnegative_int64(const char **_next,
 const char *_cdr){
  const char *next;
1371
  opus_int64  ret;
1372
1373
1374
1375
1376
1377
  int         i;
  next=_cdr+strspn(_cdr,OP_HTTP_DIGIT);
  *_next=next;
  if(OP_UNLIKELY(next<=_cdr))return OP_FALSE;
  while(*_cdr=='0')_cdr++;
  if(OP_UNLIKELY(next-_cdr>19))return OP_EIMPL;
1378
  ret=0;
1379
1380
1381
1382
  for(i=0;i<next-_cdr;i++){
    int digit;
    digit=_cdr[i]-'0';
    /*Check for overflow.*/
1383
1384
    if(OP_UNLIKELY(ret>(OP_INT64_MAX-9)/10+(digit<=7)))return OP_EIMPL;
    ret=ret*10+digit;
1385
  }
1386
  return ret;
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
}

static opus_int64 op_http_parse_content_length(const char *_cdr){
  const char *next;
  opus_int64  content_length;
  content_length=op_http_parse_nonnegative_int64(&next,_cdr);
  if(OP_UNLIKELY(*next!='\0'))return OP_FALSE;
  return content_length;
}

static int op_http_parse_content_range(opus_int64 *_first,opus_int64 *_last,
 opus_int64 *_length,const char *_cdr){
  opus_int64 first;
  opus_int64 last;
  opus_int64 length;
  size_t     d;
  if(OP_UNLIKELY(op_strncasecmp(_cdr,"bytes",5)!=0))return OP_FALSE;
  _cdr+=5;
  d=op_http_lwsspn(_cdr);
  if(OP_UNLIKELY(d<=0))return OP_FALSE;
  _cdr+=d;
  if(*_cdr!='*'){
    first=op_http_parse_nonnegative_int64(&_cdr,_cdr);
    if(OP_UNLIKELY(first<0))return (int)first;
    _cdr+=op_http_lwsspn(_cdr);
    if(*_cdr++!='-')return OP_FALSE;
    _cdr+=op_http_lwsspn(_cdr);
    last=op_http_parse_nonnegative_int64(&_cdr,_cdr);
    if(OP_UNLIKELY(last<0))return (int)last;
    _cdr+=op_http_lwsspn(_cdr);
  }
  else{
    /*This is for a 416 response (Requested range not satisfiable).*/
    first=last=-1;
    _cdr++;
  }
  if(OP_UNLIKELY(*_cdr++!='/'))return OP_FALSE;
  if(*_cdr!='*'){
    length=op_http_parse_nonnegative_int64(&_cdr,_cdr);
    if(OP_UNLIKELY(length<0))return (int)length;
  }
  else{
    /*The total length is unspecified.*/
    _cdr++;
    length=-1;
  }
  if(OP_UNLIKELY(*_cdr!='\0'))return OP_FALSE;
  if(OP_UNLIKELY(last<first))return OP_FALSE;
  if(length>=0&&OP_UNLIKELY(last>=length))return OP_FALSE;
  *_first=first;
  *_last=last;
  *_length=length;
  return 0;
}

/*Parse the Connection response header and look for a "close" token.
  Return: 1 if a "close" token is found, 0 if it's not found, and a negative
           value on error.*/
static int op_http_parse_connection(char *_cdr){
  size_t d;
  int    ret;
  ret=0;
  for(;;){
    d=strcspn(_cdr,OP_HTTP_CTOKEN);
    if(OP_UNLIKELY(d<=0))return OP_FALSE;
    if(op_strncasecmp(_cdr,"close",(int)d)==0)ret=1;
    /*We're supposed to strip and ignore any headers mentioned in the
       Connection header if this response is from an HTTP/1.0 server (to
       work around forwarding of hop-by-hop headers by old proxies), but the
       only hop-by-hop header we look at is Connection itself.
      Everything else is a well-defined end-to-end header, and going back and
       undoing the things we did based on already-examined headers would be
       hard (since we only scan them once, in a destructive manner).
      Therefore we just ignore all the other tokens.*/
    _cdr+=d;
    d=op_http_lwsspn(_cdr);
    if(d<=0)break;
    _cdr+=d;
  }
1466
  return OP_UNLIKELY(*_cdr!='\0')?OP_FALSE:ret;
1467
1468
1469
1470
1471
}

typedef int (*op_ssl_step_func)(SSL *_ssl_conn);

/*Try to run an SSL function to completion (blocking if necessary).*/
1472
static int op_do_ssl_step(SSL *_ssl_conn,op_sock _fd,op_ssl_step_func _step){
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
  struct pollfd fd;
  fd.fd=_fd;
  for(;;){
    int ret;
    int err;
    ret=(*_step)(_ssl_conn);
    if(ret>=0)return ret;
    err=SSL_get_error(_ssl_conn,ret);
    if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
    else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
    else return OP_FALSE;
    if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return OP_FALSE;
  }
}

/*Implement a BIO type that just indicates every operation should be retried.
  We use this when initializing an SSL connection via a proxy to allow the
   initial handshake to proceed all the way up to the first read attempt, and
   then return.
  This allows the TLS client hello message to be pipelined with the HTTP
   CONNECT request.*/

static int op_bio_retry_write(BIO *_b,const char *_buf,int _num){
  (void)_buf;
  (void)_num;
  BIO_clear_retry_flags(_b);
  BIO_set_retry_write(_b);
  return -1;
}

static int op_bio_retry_read(BIO *_b,char *_buf,int _num){
  (void)_buf;
  (void)_num;
  BIO_clear_retry_flags(_b);
  BIO_set_retry_read(_b);
  return -1;
}

static int op_bio_retry_puts(BIO *_b,const char *_str){
  return op_bio_retry_write(_b,_str,0);
}

static long op_bio_retry_ctrl(BIO *_b,int _cmd,long _num,void *_ptr){
  long ret;
  (void)_b;
  (void)_num;
  (void)_ptr;
  ret=0;
  switch(_cmd){
    case BIO_CTRL_RESET:
    case BIO_C_RESET_READ_REQUEST:{
      BIO_clear_retry_flags(_b);
    }
1526
    /*Fall through.*/
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
    case BIO_CTRL_EOF:
    case BIO_CTRL_SET:
    case BIO_CTRL_SET_CLOSE:
    case BIO_CTRL_FLUSH:
    case BIO_CTRL_DUP:{
      ret=1;
    }break;
  }
  return ret;
}

1538
# if (OPENSSL_VERSION_NUMBER<0x10100000L&&LIBRESSL_VERSION_NUMBER<0x2070000fL)
1539
1540
#  define BIO_set_data(_b,_ptr) ((_b)->ptr=(_ptr))
#  define BIO_set_init(_b,_init) ((_b)->init=(_init))
1541
#  define ASN1_STRING_get0_data ASN1_STRING_data
1542
1543
# endif

1544
static int op_bio_retry_new(BIO *_b){
1545
  BIO_set_init(_b,1);
1546
# if (OPENSSL_VERSION_NUMBER<0x10100000L&&LIBRESSL_VERSION_NUMBER<0x2070000fL)
1547
  _b->num=0;
Timothy B. Terriberry's avatar