add support for chunked encoding add contentlength and contentdisposition files describing the reply headers add request, content, and headers file which allows requests to be modified all changes are backward compatible with abaco and webget. Changes where done to support wdfs (a webdav/SVN client). -Steve Reference: /n/sources/patch/maybe/webfs-wdfs Date: Fri Nov 4 13:16:41 CET 2011 Signed-off-by: steve@quintile.net --- /sys/src/cmd/webfs/buf.c Fri Nov 4 13:09:16 2011 +++ /sys/src/cmd/webfs/buf.c Fri Nov 4 13:09:13 2011 @@ -28,6 +28,8 @@ n = len; memmove(buf, b->rp, n); b->rp += n; + if(b->wp == b->rp) + b->wp = b->rp = b->buf; return n; } return ioreadn(b->io, b->fd, buf, len); @@ -56,8 +58,10 @@ len--; for(p = buf;;){ + if(b->wp == b->rp && b->wp != b->buf) + b->wp = b->rp = b->buf; if(b->rp >= b->wp){ - n = ioread(b->io, b->fd, b->wp, sizeof(b->buf)/2); + n = ioread(b->io, b->fd, b->wp, &b->buf[sizeof(b->buf)] - b->wp); if(n < 0) return -1; if(n == 0) --- /sys/src/cmd/webfs/client.c Fri Nov 4 13:09:21 2011 +++ /sys/src/cmd/webfs/client.c Fri Nov 4 13:09:18 2011 @@ -47,8 +47,12 @@ (*c->url->close)(c); c->bodyopened = 0; } + free(c->contentdisposition); + c->contentdisposition = nil; free(c->contenttype); c->contenttype = nil; + free(c->contentlength); + c->contentlength = nil; free(c->postbody); c->postbody = nil; freeurl(c->url); @@ -56,6 +60,12 @@ free(c->redirect); c->redirect = nil; free(c->authenticate); + free(c->request); + c->request = nil; + free(c->headers); + c->headers = nil; + free(c->content); + c->content = nil; c->authenticate = nil; c->npostbody = 0; c->havepostbody = 0; @@ -105,12 +115,13 @@ return; } if (c->authenticate && nauth++ < 1) - continue; + continue; if(!c->redirect) break; + c->url->close(c); next = c->redirect; c->redirect = nil; - if(i==c->ctl.redirectlimit){ + if(i == c->ctl.redirectlimit){ werrstr("redirect limit reached"); goto Error; } @@ -239,8 +250,8 @@ Ctab ctltab[] = { "acceptcookies", Bool, (void*)offsetof(Ctl, acceptcookies), "sendcookies", Bool, (void*)offsetof(Ctl, sendcookies), - "redirectlimit", Int, (void*)offsetof(Ctl, redirectlimit), - "useragent", String, (void*)offsetof(Ctl, useragent), + "redirectlimit", Int, (void*)offsetof(Ctl, redirectlimit), + "useragent", String, (void*)offsetof(Ctl, useragent), }; Ctab globaltab[] = { @@ -252,8 +263,11 @@ }; Ctab clienttab[] = { - "baseurl", XUrl, (void*)offsetof(Client, baseurl), - "url", XUrl, (void*)offsetof(Client, url), + "baseurl", XUrl, (void*)offsetof(Client, baseurl), + "url", XUrl, (void*)offsetof(Client, url), + "request", String, (void*)offsetof(Client, request), + "content", String, (void*)offsetof(Client, content), + "headers", String, (void*)offsetof(Client, headers), }; static Ctab* --- /sys/src/cmd/webfs/dat.h Fri Nov 4 13:09:27 2011 +++ /sys/src/cmd/webfs/dat.h Fri Nov 4 13:09:23 2011 @@ -14,18 +14,24 @@ struct Ctl { - int acceptcookies; - int sendcookies; - int redirectlimit; - char *useragent; + int acceptcookies; + int sendcookies; + int redirectlimit; + + char *useragent; }; struct Client { - Url *url; - Url *baseurl; + Url *url; + Url *baseurl; + char *request; + char *content; + char *headers; + char *contentlength; + char *contentdisposition; Ctl ctl; - Channel *creq; /* chan(Req*) */ + Channel *creq; /* chan(Req*) */ int num; int plumbed; char *contenttype; @@ -37,6 +43,7 @@ int havepostbody; int iobusy; int bodyopened; + int chunklen; Ioproc *io; int ref; void *aux; @@ -88,7 +95,7 @@ enum { - STACK = 32*1024, /* was 16*1024; there are big arrays on the stack */ + STACK = 16384, }; extern Client** client; --- /sys/src/cmd/webfs/fs.c Fri Nov 4 13:09:33 2011 +++ /sys/src/cmd/webfs/fs.c Fri Nov 4 13:09:29 2011 @@ -45,6 +45,8 @@ Qbody, Qbodyext, Qcontenttype, + Qcontentlength, + Qcontentdisposition, Qpostbody, Qparsed, Qurl, @@ -84,23 +86,25 @@ "ctl", 0666, 0, "clone", 0666, 0, "cookies", 0666, 0, - "XXX", DMDIR|0555, 0, + "XXX", DMDIR|0555, 0, "ctl", 0666, 0, - "body", 0444, 0, - "XXX", 0444, 0, - "contenttype", 0444, 0, - "postbody", 0666, 0, + "body", 0444, 0, + "XXX", 0444, 0, + "contenttype", 0444, 0, + "contentlength", 0444, 0, + "contentdisposition", 0444, 0, + "postbody", 0666, 0, "parsed", DMDIR|0555, 0, "url", 0444, offsetof(Url, url), "scheme", 0444, offsetof(Url, scheme), - "schemedata", 0444, offsetof(Url, schemedata), - "user", 0444, offsetof(Url, user), + "schemedata", 0444, offsetof(Url, schemedata), + "user", 0444, offsetof(Url, user), "passwd", 0444, offsetof(Url, passwd), - "host", 0444, offsetof(Url, host), - "port", 0444, offsetof(Url, port), - "path", 0444, offsetof(Url, path), + "host", 0444, offsetof(Url, host), + "port", 0444, offsetof(Url, port), + "path", 0444, offsetof(Url, path), "query", 0444, offsetof(Url, query), - "fragment", 0444, offsetof(Url, fragment), + "fragment", 0444, offsetof(Url, fragment), "ftptype", 0444, offsetof(Url, ftp.type), }; @@ -194,15 +198,14 @@ static void fsread(Req *r) { - char *s; - char e[ERRMAX]; + char *s, e[ERRMAX]; Client *c; ulong path; path = r->fid->qid.path; switch(TYPE(path)){ default: - snprint(e, sizeof e, "bug in webfs path=%lux\n", path); + snprint(e, sizeof e, "bug in webfs read path=%lux\n", path); respond(r, e); break; @@ -237,6 +240,22 @@ respond(r, nil); break; + case Qcontentlength: + c = client[NUM(path)]; + if(c->contentlength == nil) + r->ofcall.count = 0; + else + readstr(r, c->contentlength); + + respond(r, nil); + break; + + case Qcontentdisposition: + c = client[NUM(path)]; + if(c->contentdisposition == nil) + r->ofcall.count = 0; + else + readstr(r, c->contentdisposition); case Qpostbody: c = client[NUM(path)]; readbuf(r, c->postbody, c->npostbody); @@ -291,7 +310,7 @@ path = r->fid->qid.path; switch(TYPE(path)){ default: - snprint(e, sizeof e, "bug in webfs path=%lux\n", path); + snprint(e, sizeof e, "bug in webfs write path=%lux\n", path); respond(r, e); break; @@ -347,6 +366,7 @@ r->ofcall.count = r->ifcall.count; respond(r, nil); break; + } } @@ -415,8 +435,26 @@ static void fsdestroyfid(Fid *fid) { + Client *c; + sendp(cclunk, fid); recvp(cclunkwait); + + switch(TYPE(fid->qid.path)){ + case Qbody: + case Qbodyext: + c = client[NUM(fid->qid.path)]; + c->bodyopened = 0; + if(c->havepostbody){ + free(c->postbody); + c->npostbody = 0; + c->postbody = nil; + c->havepostbody = 0; + } + break; + default: + break; + } } static void --- /sys/src/cmd/webfs/http.c Fri Nov 4 13:09:40 2011 +++ /sys/src/cmd/webfs/http.c Fri Nov 4 13:09:36 2011 @@ -24,6 +24,8 @@ char *netaddr; char *credentials; char autherror[ERRMAX]; + int ischunked; + uvlong length; Ibuf b; }; @@ -35,6 +37,13 @@ } static void +transferencoding(HttpState *hs, char *value) +{ + if(cistrcmp(value, "chunked") == 0) + hs->ischunked = 1; +} + +static void contenttype(HttpState *hs, char *value) { if(hs->c->contenttype != nil) @@ -43,6 +52,24 @@ } static void +contentlength(HttpState *hs, char *value) +{ + if(hs->c->contentlength != nil) + free(hs->c->contentlength); + hs->c->contentlength = estrdup(value); + hs->length = strtoull(value, nil, 0); +} + +static void +contentdisposition(HttpState *hs, char *value) +{ + if(hs->c->contentdisposition != nil) + free(hs->c->contentdisposition); + hs->c->contentdisposition = estrdup(value); +} + + +static void setcookie(HttpState *hs, char *value) { char *s, *t; @@ -117,12 +144,17 @@ up = nil; cred[0] = 0; hs->autherror[0] = 0; - if(cistrncmp(line, "basic ", 6) != 0){ + if(cistrncmp(line, "Negotiate ", 9) == 0){ + fprint(2, "skip %s\n", line); + goto error; + } + if(cistrncmp(line, "Basic ", 6) != 0){ werrstr("unknown auth: %s", line); goto error; } line += 6; if(cistrncmp(line, "realm=", 6) != 0){ + fprint(2, "ERR: missing realm: %s", line); werrstr("missing realm: %s", line); goto error; } @@ -168,7 +200,11 @@ void (*fn)(HttpState *hs, char *value); } hdrtab[] = { { "location:", location }, + { "transfer-encoding:", transferencoding }, { "content-type:", contenttype }, + { "content-type:", contenttype }, + { "content-length:", contentlength }, + { "content-disposition:", contentdisposition }, { "set-cookie:", setcookie }, { "www-authenticate:", wwwauthenticate }, }; @@ -268,7 +304,7 @@ char *cookies; Ioproc *io; HttpState *hs; - char *service; + char *request, *content, *service; if(httpdebug) fprint(2, "httpopen\n"); @@ -303,11 +339,16 @@ return -1; } hs->fd = fd; + request = "GET"; + if(c->havepostbody) + request = "POST"; + if(c->request) + request = c->request; if(httpdebug) - fprint(2, "<- %s %s HTTP/1.0\n<- Host: %s\n", - c->havepostbody? "POST": "GET", url->http.page_spec, url->host); - ioprint(io, fd, "%s %s HTTP/1.0\r\nHost: %s\r\n", - c->havepostbody? "POST" : "GET", url->http.page_spec, url->host); + fprint(2, "<- %s %s HTTP/1.1\n<- Host: %s\n", + request, url->http.page_spec, url->host); + ioprint(io, fd, "%s %s HTTP/1.1\r\nHost: %s\r\n", + request, url->http.page_spec, url->host); if(httpdebug) fprint(2, "<- User-Agent: %s\n", c->ctl.useragent); if(c->ctl.useragent) @@ -318,27 +359,40 @@ url->ischeme == UShttps); if(cookies && cookies[0]) ioprint(io, fd, "%s", cookies); - if(httpdebug) + if(cookies && cookies[0] && httpdebug) fprint(2, "<- %s", cookies); free(cookies); } + if(c->headers){ + ioprint(io, fd, "%s\r\n", c->headers); + if(httpdebug) + fprint(2, "<- %s\n", c->headers); + } if(c->havepostbody){ - ioprint(io, fd, "Content-type: %s\r\n", PostContentType); + content = PostContentType; + if(c->content) + content = c->content; + ioprint(io, fd, "Content-type: %s\r\n", content); ioprint(io, fd, "Content-length: %ud\r\n", c->npostbody); if(httpdebug){ - fprint(2, "<- Content-type: %s\n", PostContentType); + fprint(2, "<- Content-type: %s\n", content); fprint(2, "<- Content-length: %ud\n", c->npostbody); } } if(c->authenticate){ ioprint(io, fd, "Authorization: %s\r\n", c->authenticate); if(httpdebug) - fprint(2, "<- Authorization: %s\n", c->authenticate); + fprint(2, "<- Authorization: %s\r\n", c->authenticate); } ioprint(io, fd, "\r\n"); - if(c->havepostbody) + if(httpdebug) + fprint(2, "<- \r\n"); + if(c->havepostbody){ + if(httpdebug) + fprint(2, "<- %.*s", c->npostbody, c->postbody); if(iowrite(io, fd, c->postbody, c->npostbody) != c->npostbody) goto Error; + } redirect = 0; authenticate = 0; @@ -359,17 +413,15 @@ case 201: /* Created */ case 202: /* Accepted */ case 204: /* No Content */ - case 205: /* Reset Content */ + case 205: /* Reset Content */ + case 207: /* Multi status (WevDAV) */ + case 206: /* Partial Content */ #ifdef NOT_DEFINED if(ofile == nil && r->start != 0) sysfatal("page changed underfoot"); #endif break; - case 206: /* Partial Content */ - werrstr("Partial Content (206)"); - goto Error; - case 301: /* Moved Permanently */ case 302: /* Moved Temporarily */ case 303: /* See Other */ @@ -491,8 +543,7 @@ } c->authenticate = hs->credentials; hs->credentials = nil; - }else if(c->authenticate) - c->authenticate = 0; + } if(redirect){ if(!hs->location){ werrstr("redirection without Location: header"); @@ -507,10 +558,37 @@ int httpread(Client *c, Req *r) { + int n; + char buf[16]; HttpState *hs; - long n; hs = c->aux; + if(hs->ischunked){ + if(r->ifcall.offset >= hs->length){ + + if(hs->length != 0){ + /* + * all chunk size lines, except the first, + * are preceeded by a blank line. we skip it. + */ + readline(&hs->b, buf, sizeof(buf)); + } + if(readline(&hs->b, buf, sizeof(buf)) < 1){ + fprint(2, "missing next chunk length\n"); + return -1; + } + hs->length += strtoul(buf, nil, 16); + } + } + + if(hs->length){ + if(r->ifcall.offset >= hs->length) + return -1; + if(r->ifcall.count + r->ifcall.offset > hs->length) + r->ifcall.count = hs->length - r->ifcall.offset; + + } + n = readibuf(&hs->b, r->ofcall.data, r->ifcall.count); if(n < 0) return -1; --- /sys/src/cmd/webfs/main.c Fri Nov 4 13:09:45 2011 +++ /sys/src/cmd/webfs/main.c Fri Nov 4 13:09:42 2011 @@ -15,9 +15,9 @@ Ctl globalctl = { - 1, /* accept cookies */ - 1, /* send cookies */ - 10, /* redirect limit */ + 1, /* accept cookies */ + 1, /* send cookies */ + 10, /* redirect limit */ "webfs/2.0 (plan 9)" /* user agent */ }; --- /sys/src/cmd/webfs/plumb.c Fri Nov 4 13:09:51 2011 +++ /sys/src/cmd/webfs/plumb.c Fri Nov 4 13:09:48 2011 @@ -1,4 +1,5 @@ #include +#include #include #include #include --- /sys/src/cmd/webfs/url.c Fri Nov 4 13:09:59 2011 +++ /sys/src/cmd/webfs/url.c Fri Nov 4 13:09:54 2011 @@ -540,11 +540,14 @@ * in URIs to _always_ indicate escape sequences. Something like "%2500" * will still get by, but that's legitimate, and if it ends up causing * a NUL then someone is unescaping too many times. + * + * Disable this check as BitTorrent URLS can easily contain nulls + * + * if(strstr(url, "%00")){ + * werrstr("escaped NUL in URI"); + * return -1; + * } */ - if(strstr(url, "%00")){ - werrstr("escaped NUL in URI"); - return -1; - } m[0].sp = m[0].ep = nil; t = &retab[REsplit];