http.c 94.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/********************************************************************
 *                                                                  *
 * THIS FILE IS PART OF THE libopusfile SOFTWARE CODEC SOURCE CODE. *
 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
 *                                                                  *
 * THE libopusfile SOURCE CODE IS (C) COPYRIGHT 2012                *
 * by the Xiph.Org Foundation and contributors http://www.xiph.org/ *
 *                                                                  *
 ********************************************************************/
12
13
14
15
16
17
#include "internal.h"
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <string.h>

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/*RFCs referenced in this file:
  RFC 1738: Uniform Resource Locators (URL)
  RFC 1945: Hypertext Transfer Protocol -- HTTP/1.0
  RFC 2068: Hypertext Transfer Protocol -- HTTP/1.1
  RFC 2145: Use and Interpretation of HTTP Version Numbers
  RFC 2246: The TLS Protocol Version 1.0
  RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1
  RFC 2617: HTTP Authentication: Basic and Digest Access Authentication
  RFC 2817: Upgrading to TLS Within HTTP/1.1
  RFC 2818: HTTP Over TLS
  RFC 3492: Punycode: A Bootstring encoding of Unicode for Internationalized
   Domain Names in Applications (IDNA)
  RFC 3986: Uniform Resource Identifier (URI): Generic Syntax
  RFC 3987: Internationalized Resource Identifiers (IRIs)
  RFC 5894: Internationalized Domain Names for Applications (IDNA):
   Background, Explanation, and Rationale
  RFC 6066: Transport Layer Security (TLS) Extensions: Extension Definitions*/

36
37
38
39
40
41
42
43
44
45
46
typedef struct OpusParsedURL   OpusParsedURL;
typedef struct OpusStringBuf   OpusStringBuf;
typedef struct OpusHTTPConn    OpusHTTPConn;
typedef struct OpusHTTPStream  OpusHTTPStream;

static char *op_string_range_dup(const char *_start,const char *_end){
  size_t  len;
  char   *ret;
  OP_ASSERT(_start<=_end);
  len=_end-_start;
  ret=(char *)_ogg_malloc(sizeof(*ret)*(len+1));
47
48
49
50
  if(OP_LIKELY(ret!=NULL)){
    memcpy(ret,_start,sizeof(*ret)*(len));
    ret[len]='\0';
  }
51
52
53
54
55
56
57
  return ret;
}

static char *op_string_dup(const char *_s){
  return op_string_range_dup(_s,_s+strlen(_s));
}

58
59
60
61
62
63
64
65
66
67
68
static char *op_string_tolower(char *_s){
  int i;
  for(i=0;_s[i]!='\0';i++){
    int c;
    c=_s[i];
    if(c>='A'&&c<='Z')c+='a'-'A';
    _s[i]=(char)c;
  }
  return _s;
}

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
/*URI character classes (from RFC 3986).*/
#define OP_URL_ALPHA \
 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
#define OP_URL_DIGIT       "01234567890"
#define OP_URL_HEXDIGIT    "01234567890ABCDEFabcdef"
/*Not a character class, but the characters allowed in <scheme>.*/
#define OP_URL_SCHEME      OP_URL_ALPHA OP_URL_DIGIT "+-."
#define OP_URL_GEN_DELIMS  "#/:?@[]"
#define OP_URL_SUB_DELIMS  "!$&'()*+,;="
#define OP_URL_RESERVED    OP_URL_GEN_DELIMS OP_URL_SUB_DELIMS
#define OP_URL_UNRESERVED  OP_URL_ALPHA OP_URL_DIGIT "-._~"
/*Not a character class, but the characters allowed in <pct-encoded>.*/
#define OP_URL_PCT_ENCODED "%"
/*Not a character class or production rule, but for convenience.*/
#define OP_URL_PCHAR_BASE \
 OP_URL_UNRESERVED OP_URL_PCT_ENCODED OP_URL_SUB_DELIMS
#define OP_URL_PCHAR       OP_URL_PCHAR_BASE ":@"
/*Not a character class, but the characters allowed in <userinfo> and
   <IP-literal>.*/
#define OP_URL_PCHAR_NA    OP_URL_PCHAR_BASE ":"
/*Not a character class, but the characters allowed in <segment-nz-nc>.*/
#define OP_URL_PCHAR_NC    OP_URL_PCHAR_BASE "@"
/*Not a character clsss, but the characters allowed in <path>.*/
#define OP_URL_PATH        OP_URL_PCHAR "/"
/*Not a character class, but the characters allowed in <query> / <fragment>.*/
#define OP_URL_QUERY_FRAG  OP_URL_PCHAR "/?"

/*Check the <% HEXDIG HEXDIG> escapes of a URL for validity.
  Return: 0 if valid, or a negative value on failure.*/
static int op_validate_url_escapes(const char *_s){
  int i;
  for(i=0;_s[i];i++){
    if(_s[i]=='%'){
      if(OP_UNLIKELY(!isxdigit(_s[i+1]))
       ||OP_UNLIKELY(!isxdigit(_s[i+2]))
       /*RFC 3986 says %00 "should be rejected if the application is not
          expecting to receive raw data within a component."*/
       ||OP_UNLIKELY(_s[i+1]=='0'&&_s[i+2]=='0')){
        return OP_FALSE;
      }
      i+=2;
    }
  }
  return 0;
}

/*Convert a hex digit to its actual value.
  _c: The hex digit to convert.
      Presumed to be valid ('0'...'9', 'A'...'F', or 'a'...'f').
  Return: The value of the digit, in the range [0,15].*/
static int op_hex_value(int _c){
  return _c>='a'?_c-'a'+10:_c>='A'?_c-'A'+10:_c-'0';
}

/*Unescape all the <% HEXDIG HEXDIG> sequences in a string in-place.
  This does no validity checking.*/
static char *op_unescape_url_component(char *_s){
  int i;
  int j;
  for(i=j=0;_s[i];i++,j++){
    if(_s[i]=='%'){
      _s[i]=(char)(op_hex_value(_s[i+1])<<4|op_hex_value(_s[i+2]));
      i+=2;
    }
  }
  return _s;
}

/*Parse a file: URL.
  This code is not meant to be fast: strspn() with large sets is likely to be
   slow, but it is very convenient.
  It is meant to be RFC 1738-compliant (as updated by RFC 3986).*/
static const char *op_parse_file_url(const char *_src){
  const char *scheme_end;
  const char *path;
  const char *path_end;
  scheme_end=_src+strspn(_src,OP_URL_SCHEME);
  if(OP_UNLIKELY(*scheme_end!=':')
   ||scheme_end-_src!=4||op_strncasecmp(_src,"file",4)!=0){
    /*Unsupported protocol.*/
    return NULL;
  }
  /*Make sure all escape sequences are valid to simplify unescaping later.*/
  if(OP_UNLIKELY(op_validate_url_escapes(scheme_end+1)<0))return NULL;
  if(scheme_end[1]=='/'&&scheme_end[2]=='/'){
    const char *host;
    /*file: URLs can have a host!
      Yeah, I was surprised, too, but that's what RFC 1738 says.
      It also says, "The file URL scheme is unusual in that it does not specify
       an Internet protocol or access method for such files; as such, its
       utility in network protocols between hosts is limited," which is a mild
       understatement.*/
    host=scheme_end+3;
    /*The empty host is what we expect.*/
    if(OP_LIKELY(*host!='/'))path=host;
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
    else{
      const char *host_end;
      char        host_buf[28];
      /*RFC 1738 says localhost "is interpreted as `the machine from which the
         URL is being interpreted,'" so let's check for it.*/
      host_end=host+strspn(host,OP_URL_PCHAR_BASE);
      /*No <port> allowed.
        This also rejects IP-Literals.*/
      if(*host_end!='/')return NULL;
      /*An escaped "localhost" can take at most 27 characters.*/
      if(OP_UNLIKELY(host_end-host>27))return NULL;
      memcpy(host_buf,host,sizeof(*host_buf)*(host_end-host));
      host_buf[host_end-host]='\0';
      op_unescape_url_component(host_buf);
      op_string_tolower(host_buf);
      /*Some other host: give up.*/
      if(OP_UNLIKELY(strcmp(host_buf,"localhost")!=0))return NULL;
      path=host_end;
    }
183
184
185
186
187
188
189
190
191
192
193
194
195
196
  }
  else path=scheme_end+1;
  path_end=path+strspn(path,OP_URL_PATH);
  /*This will reject a <query> or <fragment> component, too.
    I don't know what to do with queries, but a temporal fragment would at
     least make sense.
    RFC 1738 pretty clearly defines a <searchpart> that's equivalent to the
     RFC 3986 <query> component for other schemes, but not the file: scheme,
     so I'm going to just reject it.*/
  if(*path_end!='\0')return NULL;
  return path;
}

#if defined(OP_ENABLE_HTTP)
197
# include <sys/ioctl.h>
198
199
# include <sys/types.h>
# include <sys/socket.h>
200
# include <sys/timeb.h>
201
# include <arpa/inet.h>
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
202
# include <netinet/in.h>
203
# include <netinet/tcp.h>
204
205
206
207
208
209
# include <fcntl.h>
# include <netdb.h>
# include <poll.h>
# include <unistd.h>
# include <openssl/ssl.h>

210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
/*The maximum number of simultaneous connections.
  RFC 2616 says this SHOULD NOT be more than 2, but everyone on the modern web
   ignores that (e.g., IE 8 bumped theirs up from 2 to 6, Firefox uses 15).
  If it makes you feel better, we'll only ever actively read from one of these
   at a time.
  The others are kept around mainly to avoid slow-starting a new connection
   when seeking, and time out rapidly.*/
# define OP_NCONNS_MAX (4)

/*The number of redirections at which we give up.
  The value here is the current default in Firefox.
  RFC 2068 mandated a maximum of 5, but RFC 2616 relaxed that to "a client
   SHOULD detect infinite redirection loops."
  Fortunately, 20 is less than infinity.*/
# define OP_REDIRECT_LIMIT (20)

/*The maximum size of a response message (before the body).
  Responses larger than this will be discarded.
  The buffer for this is currently stack-allocated, which will have to change
   if you want to make it much larger.*/
# define OP_RESPONSE_SIZE_MAX (1024)

/*The number of milliseconds we will allow a connection to sit idle before we
   refuse to resurrect it.
  Apache as of 2.2 has reduced its default timeout to 5 seconds (from 15), so
   that's what we'll use here.*/
# define OP_CONNECTION_IDLE_TIMEOUT_MS (5*1000)

/*The number of milliseconds we will wait to send or receive data before giving
   up.*/
# define OP_POLL_TIMEOUT_MS (30*1000)

/*We will always attempt to read ahead at least this much in preference to
   opening a new connection.*/
# define OP_READAHEAD_THRESH_MIN (32*(opus_int32)1024)
/*The amount to read ahead per iteration of the read-ahead loop.
  16 kB is the largest size OpenSSL will return at once.*/
# define OP_READAHEAD_CHUNK_SIZE (16*1024)

/*The amount of data to request after a seek.
  This is a trade-off between read throughput after a seek vs. the the ability
   to quickly perform another seek with the same connection.*/
# define OP_PIPELINE_CHUNK_SIZE     (32*(opus_int32)1024)
/*Subsequent chunks are requested with larger and larger sizes until they pass
   this threshold, after which we just ask for the rest of the resource.*/
# define OP_PIPELINE_CHUNK_SIZE_MAX (1024*(opus_int32)1024)
/*This is the maximum number of requests we'll make with a single connection.
  Many servers will simply disconnect after we attempt some number of requests,
   possibly without sending a Connection: close header, meaning we won't
   discover it until we try to read beyond the end of the current chunk.
  We can reconnect when that happens, but this is slow.
  Instead, we impose a limit ourselves (set to the default for Apache
   installations and thus likely the most common value in use).*/
# define OP_PIPELINE_MAX_REQUESTS   (100)
/*This should be the number of requests, starting from a chunk size of
   OP_PIPELINE_CHUNK_SIZE and doubling each time, until we exceed
   OP_PIPELINE_CHUNK_SIZE_MAX and just request the rest of the file.
  We won't reuse a connection when seeking unless it has at least this many
   requests left, to reduce the chances we'll have to open a new connection
   while reading forward afterwards.*/
# define OP_PIPELINE_MIN_REQUESTS   (7)

/*Is this an https URL?
  For now we can simply check the last letter of the scheme.*/
# define OP_URL_IS_SSL(_url) ((_url)->scheme[4]=='s')

/*Does this URL use the default port for its scheme?*/
# define OP_URL_IS_DEFAULT_PORT(_url) \
 (!OP_URL_IS_SSL(_url)&&(_url)->port==80 \
 ||OP_URL_IS_SSL(_url)&&(_url)->port==443)

281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
struct OpusParsedURL{
  /*Either "http" or "https".*/
  char     *scheme;
  /*The user name from the <userinfo> component, or NULL.*/
  char     *user;
  /*The password from the <userinfo> component, or NULL.*/
  char     *pass;
  /*The <host> component.
    This may not be NULL.*/
  char     *host;
  /*The <path> and <query> components.
    This may not be NULL.*/
  char     *path;
  /*The <port> component.
    This is set to the default port if the URL did not contain one.*/
  unsigned  port;
};

/*Parse a URL.
  This code is not meant to be fast: strspn() with large sets is likely to be
   slow, but it is very convenient.
302
303
304
305
  It is meant to be RFC 3986-compliant.
  We currently do not support IRIs (Internationalized Resource Identifiers,
   RFC 3987).
  Callers should translate them to URIs first.*/
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
static int op_parse_url_impl(OpusParsedURL *_dst,const char *_src){
  const char  *scheme_end;
  const char  *authority;
  const char  *userinfo_end;
  const char  *user;
  const char  *user_end;
  const char  *pass;
  const char  *hostport;
  const char  *hostport_end;
  const char  *host_end;
  const char  *port;
  opus_int32   port_num;
  const char  *port_end;
  const char  *path;
  const char  *path_end;
  const char  *uri_end;
  scheme_end=_src+strspn(_src,OP_URL_SCHEME);
  if(OP_UNLIKELY(*scheme_end!=':')
   ||OP_UNLIKELY(scheme_end-_src<4)||OP_UNLIKELY(scheme_end-_src>5)
   ||OP_UNLIKELY(op_strncasecmp(_src,"https",scheme_end-_src)!=0)){
    /*Unsupported protocol.*/
    return OP_EIMPL;
  }
  if(OP_UNLIKELY(scheme_end[1]!='/')||OP_UNLIKELY(scheme_end[2]!='/')){
    /*We require an <authority> component.*/
    return OP_EINVAL;
  }
  authority=scheme_end+3;
  /*Make sure all escape sequences are valid to simplify unescaping later.*/
  if(OP_UNLIKELY(op_validate_url_escapes(authority)<0))return OP_EINVAL;
  /*Look for a <userinfo> component.*/
  userinfo_end=authority+strspn(authority,OP_URL_PCHAR_NA);
  if(*userinfo_end=='@'){
    /*Found one.*/
    user=authority;
    /*Look for a password (yes, clear-text passwords are deprecated, I know,
       but what else are people supposed to use? use SSL if you care).*/
    user_end=authority+strspn(authority,OP_URL_PCHAR_BASE);
    if(*user_end==':')pass=user_end+1;
    else pass=NULL;
    hostport=userinfo_end+1;
  }
  else{
349
350
351
    /*We shouldn't have to initialize user_end, but gcc is too dumb to figure
       out that user!=NULL below means we didn't take this else branch.*/
    user=user_end=NULL;
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
    pass=NULL;
    hostport=authority;
  }
  /*Try to figure out where the <host> component ends.*/
  if(hostport[0]=='['){
    hostport++;
    /*We have an <IP-literal>, which can contain colons.*/
    hostport_end=host_end=hostport+strspn(hostport,OP_URL_PCHAR_NA);
    if(OP_UNLIKELY(*hostport_end++!=']'))return OP_EINVAL;
  }
  /*Currently we don't support IDNA (RFC 5894), because I don't want to deal
     with the policy about which domains should not be internationalized to
     avoid confusing similarities.
    Give this API Punycode (RFC 3492) domain names instead.*/
  else hostport_end=host_end=hostport+strspn(hostport,OP_URL_PCHAR_BASE);
  /*TODO: Validate host.*/
  /*Is there a port number?*/
  port_num=-1;
  if(*hostport_end==':'){
    int i;
    port=hostport_end+1;
    port_end=port+strspn(port,OP_URL_DIGIT);
    path=port_end;
    /*Not part of RFC 3986, but require port numbers in the range 0...65535.*/
    if(OP_LIKELY(port_end-port>0)){
      while(*port=='0')port++;
      if(OP_UNLIKELY(port_end-port>5))return OP_EINVAL;
      port_num=0;
      for(i=0;i<port_end-port;i++)port_num=port_num*10+port[i]-'0';
      if(OP_UNLIKELY(port_num>65535))return OP_EINVAL;
    }
  }
  else path=hostport_end;
  path_end=path+strspn(path,OP_URL_PATH);
  /*If the path is not empty, it must begin with a '/'.*/
  if(OP_LIKELY(path_end>path)&&OP_UNLIKELY(path[0]!='/'))return OP_EINVAL;
  /*Consume the <query> component, if any (right now we don't split this out
     from the <path> component).*/
  if(*path_end=='?')path_end=path_end+strspn(path_end,OP_URL_QUERY_FRAG);
  /*Discard the <fragment> component, if any.
    This doesn't get sent to the server.
    Some day we should add support for Media Fragment URIs
     <http://www.w3.org/TR/media-frags/>.*/
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
395
  if(*path_end=='#')uri_end=path_end+1+strspn(path_end+1,OP_URL_QUERY_FRAG);
396
397
  else uri_end=path_end;
  /*If there's anything left, this was not a valid URL.*/
Timothy B. Terriberry's avatar
Timothy B. Terriberry committed
398
  if(OP_UNLIKELY(*uri_end!='\0'))return OP_EINVAL;
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
  _dst->scheme=op_string_range_dup(_src,scheme_end);
  if(OP_UNLIKELY(_dst->scheme==NULL))return OP_EFAULT;
  op_string_tolower(_dst->scheme);
  if(user!=NULL){
    _dst->user=op_string_range_dup(user,user_end);
    if(OP_UNLIKELY(_dst->user==NULL))return OP_EFAULT;
    op_unescape_url_component(_dst->user);
    /*Unescaping might have created a ':' in the username.
      That's not allowed by RFC 2617's Basic Authentication Scheme.*/
    if(OP_UNLIKELY(strchr(_dst->user,':')!=NULL))return OP_EINVAL;
  }
  else _dst->user=NULL;
  if(pass!=NULL){
    _dst->pass=op_string_range_dup(pass,userinfo_end);
    if(OP_UNLIKELY(_dst->pass==NULL))return OP_EFAULT;
    op_unescape_url_component(_dst->pass);
  }
  else _dst->pass=NULL;
  _dst->host=op_string_range_dup(hostport,host_end);
  if(OP_UNLIKELY(_dst->host==NULL))return OP_EFAULT;
  if(port_num<0){
    if(_src[4]=='s')port_num=443;
    else port_num=80;
  }
  _dst->port=(unsigned)port_num;
  /*RFC 2616 says an empty <abs-path> component is equivalent to "/", and we
     MUST use the latter in the Request-URI.
    Reserve space for the slash here.*/
  if(path==path_end||path[0]=='?')path--;
  _dst->path=op_string_range_dup(path,path_end);
  if(OP_UNLIKELY(_dst->path==NULL))return OP_EFAULT;
  /*And force-set it here.*/
  _dst->path[0]='/';
  return 0;
}

static void op_parsed_url_init(OpusParsedURL *_url){
  memset(_url,0,sizeof(*_url));
}

static void op_parsed_url_clear(OpusParsedURL *_url){
  _ogg_free(_url->scheme);
  _ogg_free(_url->user);
  _ogg_free(_url->pass);
  _ogg_free(_url->host);
  _ogg_free(_url->path);
}

static int op_parse_url(OpusParsedURL *_dst,const char *_src){
  OpusParsedURL url;
  int           ret;
  op_parsed_url_init(&url);
  ret=op_parse_url_impl(&url,_src);
  if(OP_UNLIKELY(ret<0))op_parsed_url_clear(&url);
  else *_dst=*&url;
  return ret;
}

/*A buffer to hold growing strings.
  The main purpose of this is to consolidate allocation checks and simplify
   cleanup on a failed allocation.*/
struct OpusStringBuf{
  char *buf;
  int   nbuf;
  int   cbuf;
};

static void op_sb_init(OpusStringBuf *_sb){
  _sb->buf=NULL;
  _sb->nbuf=0;
  _sb->cbuf=0;
}

static void op_sb_clear(OpusStringBuf *_sb){
  _ogg_free(_sb->buf);
}

static int op_sb_ensure_capacity(OpusStringBuf *_sb,int _capacity){
  char *buf;
  int   cbuf;
  buf=_sb->buf;
  cbuf=_sb->cbuf;
  if(_capacity>=cbuf-1){
    if(OP_UNLIKELY(cbuf>=INT_MAX-1>>1))return OP_EFAULT;
    if(OP_UNLIKELY(_capacity>=INT_MAX-1))return OP_EFAULT;
    cbuf=OP_MAX(2*cbuf+1,_capacity+1);
    buf=_ogg_realloc(buf,sizeof(*buf)*cbuf);
    if(OP_UNLIKELY(buf==NULL))return OP_EFAULT;
    _sb->buf=buf;
    _sb->cbuf=cbuf;
  }
  return 0;
}

static int op_sb_append(OpusStringBuf *_sb,const char *_s,int _len){
  char *buf;
  int   nbuf;
  int   ret;
  nbuf=_sb->nbuf;
  if(OP_UNLIKELY(nbuf>INT_MAX-_len))return OP_EFAULT;
  ret=op_sb_ensure_capacity(_sb,nbuf+_len);
  if(OP_UNLIKELY(ret<0))return ret;
  buf=_sb->buf;
  memcpy(buf+nbuf,_s,sizeof(*buf)*_len);
  nbuf+=_len;
  buf[nbuf]='\0';
  _sb->nbuf=nbuf;
  return 0;
}

static int op_sb_append_string(OpusStringBuf *_sb,const char *_s){
  return op_sb_append(_sb,_s,strlen(_s));
}

513
514
515
516
517
518
519
static int op_sb_append_port(OpusStringBuf *_sb,unsigned _port){
  char port_buf[7];
  OP_ASSERT(_port<=65535U);
  sprintf(port_buf,":%u",_port);
  return op_sb_append_string(_sb,port_buf);
}

520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
static int op_sb_append_nonnegative_int64(OpusStringBuf *_sb,opus_int64 _i){
  char digit;
  int  nbuf_start;
  int  ret;
  OP_ASSERT(_i>=0);
  nbuf_start=_sb->nbuf;
  ret=0;
  do{
    digit='0'+_i%10;
    ret|=op_sb_append(_sb,&digit,1);
    _i/=10;
  }
  while(_i>0);
  if(OP_LIKELY(ret>=0)){
    char *buf;
    int   nbuf_end;
    buf=_sb->buf;
    nbuf_end=_sb->nbuf-1;
    /*We've added the digits backwards.
      Reverse them.*/
    while(nbuf_start<nbuf_end){
      digit=buf[nbuf_start];
      buf[nbuf_start]=buf[nbuf_end];
      buf[nbuf_end]=digit;
      nbuf_start++;
      nbuf_end--;
    }
  }
  return ret;
}

551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
static struct addrinfo *op_resolve(const char *_host,unsigned _port){
  struct addrinfo *addrs;
  struct addrinfo  hints;
  char             service[6];
  memset(&hints,0,sizeof(hints));
  hints.ai_socktype=SOCK_STREAM;
  OP_ASSERT(_port<=65535U);
  sprintf(service,"%u",_port);
  if(OP_LIKELY(!getaddrinfo(_host,service,&hints,&addrs)))return addrs;
  return NULL;
}

static int op_sock_set_nonblocking(int _fd,int _nonblocking){
  int flags;
  flags=fcntl(_fd,F_GETFL);
  if(OP_UNLIKELY(flags<0))return flags;
  if(_nonblocking)flags|=O_NONBLOCK;
  else flags&=~O_NONBLOCK;
  return fcntl(_fd,F_SETFL,flags);
}

/*Disable/enable write coalescing if we can.
  We always send whole requests at once and always parse the response headers
   before sending another one, so normally write coalescing just causes added
   delay.*/
static void op_sock_set_tcp_nodelay(int _fd,int _nodelay){
# if defined(TCP_NODELAY)&&(defined(IPPROTO_TCP)||defined(SOL_TCP))
#  if defined(IPPROTO_TCP)
#   define OP_SO_LEVEL IPPROTO_TCP
#  else
#   define OP_SO_LEVEL SOL_TCP
#  endif
  int ret;
  ret=setsockopt(_fd,OP_SO_LEVEL,TCP_NODELAY,&_nodelay,sizeof(_nodelay));
  /*It doesn't really matter if this call fails, but it would be interesting
     to hit a case where it does.*/
  OP_ASSERT(!ret);
# endif
}

/*A single physical connection to an HTTP server.
  We may have several of these open at once.*/
593
594
595
struct OpusHTTPConn{
  /*The current position indicator for this connection.*/
  opus_int64    pos;
596
597
598
599
600
601
602
603
604
  /*The position where the current request will end, or -1 if we're reading
     until EOF (an unseekable stream or the initial HTTP/1.0 request).*/
  opus_int64    end_pos;
  /*The position where next request we've sent will start, or -1 if we haven't
     sent the next request yet.*/
  opus_int64    next_pos;
  /*The end of the next request or -1 if we requested the rest of the resource.
    This is only set to a meaningful value if next_pos is not -1.*/
  opus_int64    next_end;
605
  /*The SSL connection, if this is https.*/
606
  SSL          *ssl_conn;
607
  /*The next connection in either the LRU or free list.*/
608
  OpusHTTPConn *next;
609
610
611
612
613
614
615
  /*The last time we blocked for reading from this connection.*/
  struct timeb  read_time;
  /*The number of bytes we've read since the last time we blocked.*/
  opus_int64    read_bytes;
  /*The estimated throughput of this connection, in bytes/s.*/
  opus_int64    read_rate;
  /*The socket we're reading from.*/
616
  int           fd;
617
618
619
620
  /*The number of remaining requests we are allowed on this connection.*/
  int           nrequests_left;
  /*The chunk size to use for pipelining requests.*/
  opus_int32    chunk_size;
621
622
623
};

static void op_http_conn_init(OpusHTTPConn *_conn){
624
  _conn->next_pos=-1;
625
626
627
628
629
630
631
632
  _conn->ssl_conn=NULL;
  _conn->next=NULL;
  _conn->fd=-1;
}

static void op_http_conn_clear(OpusHTTPConn *_conn){
  if(_conn->ssl_conn!=NULL)SSL_free(_conn->ssl_conn);
  /*SSL frees the BIO for us.*/
633
  if(_conn->fd>=0)close(_conn->fd);
634
635
}

636
/*The global stream state.*/
637
638
639
640
641
struct OpusHTTPStream{
  /*The list of connections.*/
  OpusHTTPConn     conns[OP_NCONNS_MAX];
  /*The context object used as a framework for TLS/SSL functions.*/
  SSL_CTX         *ssl_ctx;
642
643
  /*The cached session to reuse for future connections.*/
  SSL_SESSION     *ssl_session;
644
645
  /*The LRU list (ordered from MRU to LRU) of currently connected
     connections.*/
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
  OpusHTTPConn    *lru_head;
  /*The free list.*/
  OpusHTTPConn    *free_head;
  /*The URL to connect to.*/
  OpusParsedURL    url;
  /*Information about the address we connected to.*/
  struct addrinfo  addr_info;
  /*The address we connected to.*/
  union{
    struct sockaddr     s;
    struct sockaddr_in  v4;
    struct sockaddr_in6 v6;
  }                addr;
  /*A buffer used to build HTTP requests.*/
  OpusStringBuf    request;
661
662
  /*A buffer used to build proxy CONNECT requests.*/
  OpusStringBuf    proxy_connect;
663
  /*The Content-Length, if specified, or -1 otherwise.
664
    This will always be specified for seekable streams.*/
665
666
667
  opus_int64       content_length;
  /*The position indicator used when no connection is active.*/
  opus_int64       pos;
668
669
670
671
672
673
674
675
676
677
678
679
680
  /*The connection we're currently reading from.
    This can be -1 if no connection is active.*/
  int              cur_conni;
  /*Whether or not the server supports range requests.*/
  int              seekable;
  /*Whether or not the server supports HTTP/1.1 with persistent connections.*/
  int              pipeline;
  /*The offset of the tail of the request.
    Only the offset in the Range: header appears after this, allowing us to
     quickly edit the request to ask for a new range.*/
  int              request_tail;
  /*The estimated time required to open a new connection, in milliseconds.*/
  opus_int32       connect_rate;
681
682
683
684
685
686
687
688
689
690
691
692
};

static void op_http_stream_init(OpusHTTPStream *_stream){
  OpusHTTPConn **pnext;
  int ci;
  pnext=&_stream->free_head;
  for(ci=0;ci<OP_NCONNS_MAX;ci++){
    op_http_conn_init(_stream->conns+ci);
    *pnext=_stream->conns+ci;
    pnext=&_stream->conns[ci].next;
  }
  _stream->ssl_ctx=NULL;
693
  _stream->ssl_session=NULL;
694
695
  _stream->lru_head=NULL;
  op_parsed_url_init(&_stream->url);
696
  op_sb_init(&_stream->request);
697
  op_sb_init(&_stream->proxy_connect);
698
699
700
  _stream->seekable=0;
}

701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
/*Close the connection and move it to the free list.
  _stream:     The stream containing the free list.
  _conn:       The connection to close.
  _penxt:      The linked-list pointer currently pointing to this connection.
  _gracefully: Whether or not to shut down cleanly.*/
static void op_http_conn_close(OpusHTTPStream *_stream,OpusHTTPConn *_conn,
 OpusHTTPConn **_pnext,int _gracefully){
  /*If we don't shut down gracefully, the server MUST NOT re-use our session
     according to RFC 2246, because it can't tell the difference between an
     abrupt close and a truncation attack.
    So we shut down gracefully if we can.
    However, we will not wait if this would block (it's not worth the savings
     from session resumption to do so).
    Clients (that's us) MAY resume a TLS session that ended with an incomplete
     close, according to RFC 2818, so that's no reason to make sure the server
     shut things down gracefully.
    It also says "client implementations MUST treat any premature closes as
     errors and the data received as potentially truncated," but libopusfile
     treats errors and potentially truncated data in unseekable streams just
     like a normal EOF.
    We warn about this in the docs, and give some suggestions if you truly want
     to avoid truncation attacks.*/
  if(_gracefully&&_conn->ssl_conn!=NULL)SSL_shutdown(_conn->ssl_conn);
724
  op_http_conn_clear(_conn);
725
  _conn->next_pos=-1;
726
727
  _conn->ssl_conn=NULL;
  _conn->fd=-1;
728
729
  OP_ASSERT(*_pnext==_conn);
  *_pnext=_conn->next;
730
731
732
733
734
  _conn->next=_stream->free_head;
  _stream->free_head=_conn;
}

static void op_http_stream_clear(OpusHTTPStream *_stream){
735
736
737
  while(_stream->lru_head!=NULL){
    op_http_conn_close(_stream,_stream->lru_head,&_stream->lru_head,0);
  }
738
  if(_stream->ssl_session!=NULL)SSL_SESSION_free(_stream->ssl_session);
739
  if(_stream->ssl_ctx!=NULL)SSL_CTX_free(_stream->ssl_ctx);
740
  op_sb_clear(&_stream->proxy_connect);
741
742
743
744
  op_sb_clear(&_stream->request);
  op_parsed_url_clear(&_stream->url);
}

745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
static int op_http_conn_write_fully(OpusHTTPConn *_conn,
 const char *_buf,int _size){
  struct pollfd  fd;
  SSL           *ssl_conn;
  fd.fd=_conn->fd;
  ssl_conn=_conn->ssl_conn;
  while(_size>0){
    int err;
    if(ssl_conn!=NULL){
      int ret;
      ret=SSL_write(ssl_conn,_buf,_size);
      if(ret>0){
        /*Wrote some data.*/
        _buf+=ret;
        _size-=ret;
        continue;
      }
      /*Connection closed.*/
      else if(ret==0)return OP_FALSE;
      err=SSL_get_error(ssl_conn,ret);
      /*Yes, renegotiations can cause SSL_write() to block for reading.*/
      if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
      else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
      else return OP_FALSE;
    }
    else{
      ssize_t ret;
      errno=0;
      ret=write(fd.fd,_buf,_size);
      if(ret>0){
        _buf+=ret;
        _size-=ret;
        continue;
      }
      err=errno;
      if(err!=EAGAIN&&err!=EWOULDBLOCK)return OP_FALSE;
      fd.events=POLLOUT;
    }
    if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return OP_FALSE;
  }
  return 0;
786
787
}

788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
static int op_http_conn_estimate_available(OpusHTTPConn *_conn){
  int available;
  int ret;
  ret=ioctl(_conn->fd,FIONREAD,&available);
  if(ret<0)available=0;
  /*This requires the SSL read_ahead flag to be unset to work.
    We ignore partial records as well as the protocol overhead for any pending
     bytes.
    This means we might return somewhat less than can truly be read without
     blocking (if there's a partial record).
    This is okay, because we're using this value to estimate network transfer
     time, and we _have_ already received those bytes.
    We also might return slightly more (due to protocol overhead), but that's
     small enough that it probably doesn't matter.*/
  if(_conn->ssl_conn!=NULL)available+=SSL_pending(_conn->ssl_conn);
  return available;
804
805
}

806
807
808
809
810
811
812
813
814
static opus_int32 op_time_diff_ms(const struct timeb *_end,
 const struct timeb *_start){
  opus_int64 dtime;
  dtime=_end->time-_start->time;
  OP_ASSERT(_end->millitm<1000);
  OP_ASSERT(_start->millitm<1000);
  if(OP_UNLIKELY(dtime>(0x7FFFFFFF-1000)/1000))return 0x7FFFFFFF;
  if(OP_UNLIKELY(dtime<(-0x7FFFFFFF+999)/1000))return -0x7FFFFFFF-1;
  return (opus_int32)dtime*1000+_end->millitm-_start->millitm;
815
816
}

817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
/*Update the read rate estimate for this connection.*/
static void op_http_conn_read_rate_update(OpusHTTPConn *_conn){
  struct timeb read_time;
  opus_int32   read_delta_ms;
  opus_int64   read_delta_bytes;
  opus_int64   read_rate;
  int          ret;
  read_delta_bytes=_conn->read_bytes;
  if(read_delta_bytes<=0)return;
  ret=ftime(&read_time);
  OP_ASSERT(!ret);
  read_delta_ms=op_time_diff_ms(&read_time,&_conn->read_time);
  read_rate=_conn->read_rate;
  read_delta_ms=OP_MAX(read_delta_ms,1);
  read_rate+=read_delta_bytes*1000/read_delta_ms-read_rate+4>>3;
  *&_conn->read_time=*&read_time;
  _conn->read_bytes=0;
  _conn->read_rate=read_rate;
}
836

837
838
839
840
841
842
843
844
845
846
847
848
849
850
/*Tries to read from the given connection.
  [out] _buf: Returns the data read.
  _size:      The size of the buffer.
  _blocking:  Whether or not to block until some data is retrieved.*/
static ptrdiff_t op_http_conn_read(OpusHTTPConn *_conn,
 char *_buf,ptrdiff_t _size,int _blocking){
  struct pollfd   fd;
  SSL            *ssl_conn;
  ptrdiff_t       nread;
  ptrdiff_t       nread_unblocked;
  fd.fd=_conn->fd;
  ssl_conn=_conn->ssl_conn;
  nread=nread_unblocked=0;
  do{
851
    int err;
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
    if(ssl_conn!=NULL){
      int ret;
      ret=SSL_read(ssl_conn,_buf+nread,_size-nread);
      if(ret>0){
        /*Read some data.
          Keep going to see if there's more.*/
        nread+=ret;
        nread_unblocked+=ret;
        continue;
      }
      /*Connection closed.*/
      else if(ret==0)break;
      /*If we already read some data, return it right now.*/
      if(nread>0)break;
      err=SSL_get_error(ssl_conn,ret);
      if(err==SSL_ERROR_WANT_READ)fd.events=POLLIN;
      /*Yes, renegotiations can cause SSL_read() to block for writing.*/
      else if(err==SSL_ERROR_WANT_WRITE)fd.events=POLLOUT;
      else return 0;
    }
    else{
      ssize_t ret;
      errno=0;
      ret=read(fd.fd,_buf+nread,_size-nread);
      if(ret>0){
        /*Read some data.
          Keep going to see if there's more.*/
        nread+=ret;
        nread_unblocked+=ret;
        continue;
      }
      /*If we already read some data, return it right now.*/
      if(nread>0)break;
      err=errno;
      if(err!=EAGAIN&&err!=EWOULDBLOCK)return 0;
      fd.events=POLLIN;
    }
    _conn->read_bytes+=nread_unblocked;
    op_http_conn_read_rate_update(_conn);
    nread_unblocked=0;
    if(!_blocking)break;
    /*Need to wait to get any data at all.*/
    if(poll(&fd,1,OP_POLL_TIMEOUT_MS)<=0)return 0;
895
896
  }
  while(nread<_size);
897
  _conn->read_bytes+=nread_unblocked;
898
899
900
901
  return nread;
}

/*Reads the entirety of a response to an HTTP request into a buffer.
902
903
904
905
906
907
908
  Actual parsing and validation is done later.
  _buf:  The buffer in which to read the response.
         No terminating NUL is appended.
  _size: The number of bytes available in the buffer.
  Return: The number of bytes in the response on success, OP_EREAD if the
           connection was closed before reading any data, or another negative
           value on any other error.*/
909
910
911
912
913
914
915
916
917
918
919
920
static int op_http_conn_read_response(OpusHTTPConn *_conn,
 char *_buf,int _size){
  /*The remaining size of the buffer.*/
  int size;
  /*How many characters we've yet to see from the "\r\n\r\n" terminator.*/
  int state;
  size=_size;
  state=4;
  while(size>=state){
    ptrdiff_t ret;
    int       len;
    ret=op_http_conn_read(_conn,_buf,state,1);
921
    if(ret<=0)return _size==size?OP_EREAD:OP_FALSE;
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
    /*We read some data.*/
    _buf+=ret;
    size-=ret;
    len=_size-size;
    /*Make sure the starting characters are "HTTP".
      Otherwise we could wind up waiting forever for a response from
       something that is not an HTTP server.*/
    if(len-ret<4&&op_strncasecmp(_buf-len,"HTTP",OP_MIN(len,4))!=0){
      return OP_FALSE;
    }
    /*How far along on the "\r\n\r\n" terminator are we?*/
    if(*(_buf-1)=='\n'){
      if(len>=2&&*(_buf-2)=='\r'){
        if(len>=4&&*(_buf-3)=='\n'&&*(_buf-4)=='\r')return len;
        state=2;
      }
      else state=4;
    }
    else if(*(_buf-1)=='\r'){
      state=3;
      if(len>=3&&*(_buf-2)=='\n'&&*(_buf-3)=='\r')state=1;
    }
    else state=4;
  }
  /*Not enough space left in the buffer to add the characters we'd need to get
     a valid terminator.*/
  return OP_EIMPL;
}

# define OP_HTTP_DIGIT "01234567890"

/*The Reason-Phrase is not allowed to contain control characters, except
954
   horizontal tab (HT: \011).*/
955
956
957
958
959
960
961
962
# define OP_HTTP_CREASON_PHRASE \
 "\001\002\003\004\005\006\007\010\012\013\014\015\016\017\020\021" \
 "\022\023\024\025\026\027\030\031\032\033\034\035\036\037\177"

# define OP_HTTP_CTLS \
 "\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020" \
 "\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037\177"

963
964
/*This also includes '\t', but we get that from OP_HTTP_CTLS.*/
# define OP_HTTP_SEPARATORS " \"(),/:;<=>?@[\\]{}"
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979

/*TEXT can also include LWS, but that has structure, so we parse it
   separately.*/
# define OP_HTTP_CTOKEN OP_HTTP_CTLS OP_HTTP_SEPARATORS

/*Return: The amount of linear white space (LWS) at the start of _s.*/
static int op_http_lwsspn(const char *_s){
  int i;
  for(i=0;;){
    if(_s[0]=='\r'&&_s[1]=='\n'&&(_s[2]=='\t'||_s[2]==' '))i+=3;
    else if(_s[i]=='\t'||_s[i]==' ')i++;
    else return i;
  }
}

980
981
static char *op_http_parse_status_line(int *_v1_1_compat,
 char **_status_code,char *_response){
982
983
  char   *next;
  char   *status_code;
984
  int     v1_1_compat;
985
986
987
988
989
990
991
992
993
994
  size_t  d;
  /*RFC 2616 Section 6.1 does not say that the tokens in the Status-Line cannot
     be separated by optional LWS, but since it specifically calls out where
     spaces are to be placed and that CR and LF are not allowed except at the
     end, I am assuming this to be true.*/
  /*We already validated that this starts with "HTTP"*/
  OP_ASSERT(op_strncasecmp(_response,"HTTP",4)==0);
  next=_response+4;
  if(OP_UNLIKELY(*next++!='/'))return NULL;
  d=strspn(next,OP_HTTP_DIGIT);
995
996
997
998
999
1000
  /*"Leading zeros MUST be ignored by recipients."*/
  while(*next=='0'){
    next++;
    OP_ASSERT(d>0);
    d--;
  }