Difference between revisions of "MHTTP"
From Scriptwiki
m (+http.close to example) |
m (Adding known issue about not supporting arrays in POST) |
||
(One intermediate revision by the same user not shown) | |||
Line 59: | Line 59: | ||
; | ; | ||
; Known issues: | ; Known issues: | ||
− | ; Cookies do not respect attributes such as domain, path, expiration, or secure. They | + | ; POSTing arrays is not currently supported |
+ | ; Cookies do not respect attributes such as domain, path, expiration, or secure. They are always sent | ||
; Relative redirects don't work, only absolute | ; Relative redirects don't work, only absolute | ||
; Redirects from insecure to secure links when you don't have SSL will not be handled well | ; Redirects from insecure to secure links when you don't have SSL will not be handled well | ||
Line 165: | Line 166: | ||
.hdel %inname chunkleft | .hdel %inname chunkleft | ||
; Get POST binvar | ; Get POST binvar | ||
− | var %postBV = $http.postdata(% | + | var %postBV = $http.postdata(%exname) |
; Use GET if there's no post data, POST otherwise | ; Use GET if there's no post data, POST otherwise | ||
if ($bvar(%postBV,0) == 0) { | if ($bvar(%postBV,0) == 0) { |
Latest revision as of 22:02, 10 June 2015
Often times you may want to download a web page from the internet using HTTP, but HTTP can be complicated. This script handles most of the basics for you including
- URL Parsing
- SSL
- Cookies
- Sending POST data
- Following redirects
- Handling CHUNKED encoding
Typical usage simply involves using the /http.open, save and close commands to download a file, and creating a signal event to do something with it once it is done.
Example
alias DownloadGooglesLogo { http.open google https://www.google.com/images/srpr/logo11w.png http.save -f google logo.png } on *:SIGNAL:http: { if ($1 == google) { if ($2 == SAVED) { run $3- | http.close $1 } if ($2 == PROGRESS) { echo -atg Download progress: $bytes($3).suf $+ / $+ $bytes($4).suf downloaded } } }
Script
; Ben's (ben@st0rm.net) HTTP for mIRC script (mHTTP) ; Used to download over HTTP ; Basic usage ; /http.open [name] <URL> ; Opens an HTTP handle for the given URL ; /http.save [-f] <name> <file|bvar> ; Downloads a given HTTP resource to the given file or binvar ; -f forces the file to be overwritten ; /http.close <name> ; Closes and HTTP handle for the given URL ; ; Here are some things you can do after opening a handle, but before saving it ; Manage POST data: ; /http.addpost <handle> <variable> <value> ; /http.delpost <handle> <variable> ; Manage cookies ; /http.addcookie <handle> <value> ; $http.cookie(handle,N) - gets Nth cookie ; ; After a request completes (e.g. after a save) you can get some data: ; $http.responseheader(handle, header, N) - Returns value of given responseheader, or $false. Only useful after a request is made ; ; You can mark a socket with data of your choice (it's a transparent write to a hashtable) similar to sockmark ; /http.mark <handle> [data] - Writes the data. Specify no data to delete the mark ; $http.mark(handle) - Reads the data ; ; Signals ; In order to tell via script when something happens (your file is being saved, for example) use ; on *:SIGNAL:http: ; where $1 = Handle name, $2 = Event name, $3- = parameters ; Events: ; COMPLETED <statuscode> - when a request completes ; SAVED <location> - when a save completes ; REDIRECT <location> - when a request is redirected elsewhere ; PROGRESS <bytes> [total size] - When part but not all of the data is downloaded. Total size is not available for chunked transfer ; ; ; Known issues: ; POSTing arrays is not currently supported ; Cookies do not respect attributes such as domain, path, expiration, or secure. They are always sent ; Relative redirects don't work, only absolute ; Redirects from insecure to secure links when you don't have SSL will not be handled well ; Creates a request that when completed is saved to a file alias http.save { if ($1 == -f) { var %force = $true tokenize 32 $2- } else { var %force = $false } if ($0 < 2) { echo -atg * /http.save: Use /http.save <http handle> <file> return } var %hashtable = http. $+ $1 if (!$hget(%hashtable)) { echo -atg * /http.save: No such HTTP handle exists. Use /http.open first return } if ($sock(%hashtable)) { echo -atg * /http.save: There is a pending request for this HTTP handle already. A connection is open the the remote host. Close it, or wait for it to complete return } var %destination = $2- if ($exists(%destination)) { if (%force) { .remove %destination } else { echo -atg * /http.save: File already exists. Try /http.save -f to force an overwrite return } } if (&* iswm %destination) { .hadd %hashtable bvarout %destination } else { ; Open file handler stream for writing .fopen -n %hashtable %destination if ($ferr) { echo -atg * /http.save: Could not use file. Error: $ferr return } ; Store fstream name in hashtable to indicate that we want to save this once it completes .hadd %hashtable fstream %hashtable } ; Set redirect counter to 0 .hadd %hashtable redirects 0 ; Start socket http.sockstart %hashtable } alias http.mark { var %hashtable = http. $+ $1 if ($isid) { ; As an identifier, read the mark return $hget(%hashtable,mark) } else { ; As a command, set the mark if (!$hget(%hashtable)) { echo -atg * /http.mark - No such handle $1 } elseif (!$2) { .hdel %hashtable mark } else { .hadd %hashtable mark $2- } } } ; Opens socket alias -l http.sockstart { if ($hget($1,secure) == $true) { if ($sock($1) && $sock($1).ssl) { http.sendrequest $1 } else { if ($sock($1)) { .sockclose $1 } if (!$sslready) { echo -atg HTTP Error - request $qt($right($1,-5)) would require you to use SSL but you are not SSL ready. Please see http://mirc.com/ssl.html http.close $right($1,-5) return } sockopen -e $1 $hget($1, host) $hget($1, port) } } else { if ($sock($1)) { http.sendrequest $1 } else { sockopen $1 $hget($1, host) $hget($1, port) } } } ; Sends the request. Assumes the socket is open and ready alias -l http.sendrequest { if (!$sock($1)) { echo -atg HTTP Internal error. Socket not open in /http.sendrequest | return } var %inname = $1 var %exname = $right($1,-5) ; Cleanup from previous requests .hdel %inname responseheaders .hdel %inname body .hdel %inname contentlength .hdel %inname chunkleft ; Get POST binvar var %postBV = $http.postdata(%exname) ; Use GET if there's no post data, POST otherwise if ($bvar(%postBV,0) == 0) { sockwrite -n %inname GET $hget(%inname,path) HTTP/1.1 } else { sockwrite -n %inname POST $hget(%inname,path) HTTP/1.1 } ; Send the host sockwrite -n %inname Host: $hget(%inname,host) ; Send the user-agent sockwrite -n %inname User-agent: $hget(%inname,user-agent) sockwrite -n %inname Connection: Keep-Alive ; Only plain-text supported sockwrite -n %inname Accept: text/plain; q=0.5, text/html ; Send cookies if ($http.cookie(%exname, 0) > 0) { sockwrite %inname Cookie: $+ $chr(32) var %i = 1 while ($http.cookie(%exname, %i)) { sockwrite %inname $http.urlencode($ifmatch) $+ ; inc %i } sockwrite -n %inname } ; If POST send content type and length if ($bvar(%postBV,0) > 0) { sockwrite -n %inname Content-Type: application/x-www-form-urlencoded sockwrite -n %inname Content-Length: $v1 } ; End request headers with empty line sockwrite -n %inname ; If POST, send data if ($bvar(%postBV,0) > 0) { sockwrite %inname %postBV ; sockwrite -n %inname } } ; Just some bookeeping in case we miss it on *:SOCKCLOSE:http.*: { if ($fopen($sockname)) { .fclose $sockname } } on *:SOCKOPEN:http.*: { http.sendrequest $sockname } on *:SOCKREAD:http.*: { if ($sockerr > 0) { echo -atg * HTTP handle $qt($right($sockname,-5)) failed with socket error $sockerr http.close $right($sockname,-5) return } ;Read all the data into &bvTemp :read ; First, read this buffer sockread &bvThis if ($sockbr > 0) { ; If data was read, copy it to the on-going buffer and continue bcopy &bvTemp $calc($bvar(&bvTemp,0) +1) &bvThis 1 -1 goto read } if ($bvar(&bvTemp,0) == 0) { echo -atg HTTP sockread called with no data to read... | return } ; Do we need to parse headers? ; If response headers not set, this call has the headers if (!$hget($sockname,responseheaders)) { ; Find the response headers var %end = $bfind(&bvTemp,1, 13 10 13 10) if (%end == 0) { echo -atg Error: No headers found, but no content length so this can't be a subsequent call | return } ; Increase to include the 3 extra characters inc %end 3 bcopy &bvResponseHeaders 1 &bvTemp 1 %end ; If there's still non-header data copy it over if (%end < $bvar(&bvTemp, 0)) { bcopy -c &bvTemp 1 &bvTemp $calc(%end + 1) -1 } else { bunset &bvTemp } ; If this is the first header response, save in hash table if (!$hget($sockname, responseheaders)) { .hadd -b $sockname responseheaders &bvResponseHeaders http.OnHeaders $sockname } } ; there's no more to read, bail if (!$bvar(&bvTemp,0)) { ; If that's all this response has to offer, we're done if ($hget($sockname,contentlength) == 0) { http.ondone $sockname } return } ; Read body var %contentlength = $hget($sockname,contentlength) if (%contentlength == $null || %contentlength == $false) { var %chunked = $true } else { var %chunked = $false } ; Set bvBody to the body ready so far if ($hget($sockname,body)) { noop $hget($sockname, body, &bvBody) } if (%chunked) { :readChunk var %chunkleft = $hget($sockname, chunkleft) if (!%chunkleft) { ; If we've read all of a chunk, or this is the first one, get its size var %crlf = $bfind(&bvTemp, 1, 13 10) if (!%crlf) { echo -atg HTTP internal error. Handle $qt($right($sockname,-5)) is in chunked encoding but no chunk size found in data return } var %chunkleft = $base($bvar(&bvTemp, 1, $calc(%crlf - 1)).text,16,10) if (%chunkleft !isnum) { echo -atg HTTP internal error. Handle $qt($right($sockname,-5)) has invalid chunk size: %chunkleft } ; New chunk else { ; Save chunk size and remove it (plus its \r\n) from temp .hadd $sockname chunkleft %chunkleft .bcopy -c &bvTemp 1 &bvTemp $calc(%crlf +2) -1 if (%chunkLeft == 0) { ; if we just read chunk size 0, that was the end. .hadd -b $sockname body &bvBody http.onDone $sockname return } } } ; read %chunkleft bytes var %bytesToRead = $iif(%chunkLeft < $bvar(&bvTemp,0),%chunkLeft, $bvar(&bvTemp,0)) .bcopy &bvBody $calc($bvar(&bvBody,0) +1) &bvTemp 1 %bytesToRead hdec $sockname chunkleft %bytesToRead .signal http $right($sockname,-5) PROGRESS %bytesToRead if (%bytesToRead == %chunkLeft) { ; If this is the end of the chunk, consume crlf inc %bytesToRead 2 ; If there's still stuff in this var, go back and read the new chunk if (%bytesToRead < $bvar(&bvTemp,0)) { .bcopy -c &bvTemp 1 &bvTemp $calc(%bytesToRead + 1) -1 goto readChunk } } ; End of this call, save body in hashtable for future calls .hadd -b $sockname body &bvBody } else { ; Copy everything .bcopy &bvBody $calc($bvar(&bvBody,0) +1) &bvTemp 1 -1 ; Add read-so-far body to hashtable .hadd -b $sockname body &bvBody .signal http $right($sockname,-5) PROGRESS $bvar(&bvBody,0) $hget($sockname,contentlength) ; Check if we're done if ($bvar(&bvBody,0) >= $hget($sockname,contentlength)) { ; We're done http.onDone $sockname if ($bvar(&bvBody,0) > $hget($sockname,contentlength)) { echo -qatg HTTP Warning: Read past content-length. } } } } ; Internal event called when we get the first header response from the server alias -l http.OnHeaders { noop $hget($1,responseheaders, &bvResponseHeaders) ; First header is always the status response var %end = $bfind(&bvResponseHeaders, 1, 13 10) if (%end <= 1) { echo -atg Error: HTTP $qt($right($1,-5)) Server did not respond with HTTP status goto cleanup } var %status = $bvar(&bvResponseHeaders,1,$calc(%end - 1)).text .hadd $1 responsestatus %status var %version = $gettok(%status,1,32) if (%version != HTTP/1.1 && %version != HTTP/1.0) { echo -atg Error: HTTP $qt($right($1,-5)) Server indicated unknown version: %version goto cleanup } var %statuscode = $gettok(%status,2,32) if (%statuscode !isnum) { echo -atg Error: HTTP $qt($right($1,-5)) Server indicated invalid status code: %statuscode goto cleanup } ; Check transfer type if ($http.responseheader($right($1,-5),Content-Length,1) != $null) { hadd $1 contentlength $v1 } elseif ($http.responseheader($right($1,-5), Transfer-Encoding, 1) != chunked) { echo -atg HTTP Handle $qt($right($1,-5)) has no content-length header and is not chunked echo -atg $bvar(&bvResponseHeaders,1-).text return } ; Add any cookies that were set var %cookie = 1 while ($http.responseheader($right($1,-5), Set-Cookie, %cookie)) { inc %cookie var %cookieValue = $gettok($ifmatch,1,$asc(;)) http.addcookie $right($1,-5) %cookieValue } ; Handle redirects if (%statuscode == 303 || %statuscode == 302 || %statuscode == 307) { var %location = $http.responseheader($right($1,-5), Location) .signal http $right($1,-5) REDIRECT %location http.seturl $1 %location hinc $1 redirects ; If the server doesn't promise us this connection is keep-alive, close it and use a new one if ($http.responseheader($right($1,-5), Connection, 1) != Keep-Alive) { if ($sock($1)) { .sockclose $1 } } if ($hget($1, redirects) > 5) { echo -atg HTTP Handle $qt($right($1,-5)) redirect loop detected return } http.sockstart $1 } return :cleanup if ($sock($1)) { .sockclose $1 } } alias http.addcookie { var %hashtable = http. $+ $1 if (!$hget(%hashtable)) { echo -atg * /http.addcookie: No such HTTP handle return } if ($0 < 2) { echo -atg * /http.addcookie: Usage: /http.addcookie <handle> <value> return } var %i = 1 var %cookieNum = 1 while ($hget(%hashtable,%i).item) { var %item = $ifmatch inc %i if (cookie-* iswm %item) { var %tmp = $gettok(%item,2,$asc(-)) ; If the value is the same, overwrite if ($2- == $hget(%hashtable,%item)) { %cookieNum = %tmp break } ; Otherwise store max cookie num elseif (%tmp >= %cookieNum) { %cookieNum = $calc(%tmp + 1) } } } .hadd %hashtable cookie- $+ %cookieNum $+ -value $2- } ; Gets an HTTP cookie for the given handle ; $http.cookie(<handle>, <index>) alias http.cookie { if ($2 !isnum) { echo -atg * $!http.cookie invalid parameters return } var %hashtable = http. $+ $1 if (!$hget(%hashtable)) { echo -atg * $!http.cookie no such HTTP handle return } var %i = 1 var %found = 0 while ($hget(%hashtable, %i).item) { inc %i var %item = $ifmatch if (cookie-*-value iswm %item) { inc %found var %id = $gettok(%item,2,$asc(-)) if (%found == $2) { return $hget(%hashtable, cookie- $+ %id $+ -value) } } } if ($2 == 0) { return %found } } ; Returns the value of the given response header ; Case sensitive since it is binary variable operation alias http.responseheader { if ($0 < 2) { echo -atg Invalid usage. Use: $http.responseheader(<handle>, <header>[, index]) return } ; %index is parameter 3 - 1-N or 0 for number var %index = 1 if ($3 isnum) { %index = $3 } ; Veriy hash table exists var %table = http. $+ $1 if (!$hget(%table,responseheaders)) { return $false } noop $hget(%table,responseheaders, &bvResponseHeaders)) ; Set up parameters for the header search ; Offset is where we start searching from in the loop ; thisIndex is the number of matches found so far var %offset = 1 var %thisIndex = 0 ; The main search loop :search var %start = $bfind(&bvResponseHeaders, %offset, $2 $+ :).text ; If there isn't another match, either the index was too large or they want the count if (!%start) { if (%index == 0) { return %thisIndex } else { return $false } } ; Find the end of the header var %end = $bfind(&bvResponseHeaders, %start, 13 10) if (%end <= %start) { echo -atg HTTP Internal error ($http.responseheader), header $qt($2) for handle $qt($1) has no CRLF termination | return $false } inc %thisIndex ; If this not the one they asked for, try again starting from the end of this header if (%index != %thisIndex) { var %offset = %end goto search } ; Find the colon. If there isn't one we'll return the whole string if ($calc($bfind(&bvResponseHeaders, %start, $asc(:)) +1) < %end) { %start = $v1 } return $bvar(&bvResponseHeaders,%start, $calc(%end - %start)).text } ; Internal event called when a request is filled. $1 = internal handle name alias -l http.onDone { if ($hget($1,responseheaders)) { noop $hget($1,responseheaders,&bvResponseHeaders) } ; Write out and close filestream if set if ($hget($1,fstream)) { noop $hget($1,body,&bvBody) if (!$bvar(&bvBody,0)) { .fclose $hget($1,fstream) } else { .fwrite -b $hget($1,fstream) &bvBody ; Save fname so we can use it in the signal after the close var %fname = $fopen($hget($1,fstream)).fname .fclose $hget($1,fstream) .signal http $right($1,-5) SAVED %fname } } ; If socket is still around, close it if ($sock($1)) { .sockclose $1 } ; Reset redirects for future calls .hadd $1 redirects 0 .signal HTTP $right($1,-5) COMPLETED $hget($1,responsestatus) if ($hget($1, bvarout)) { noop $hget($1,body,&bvBody) var %bvarout = $ifmatch bcopy %bvarout 1 &bvBody 1 -1 ; Need to use -n so that the binvar is still in scope ; So we need to do this last .signal -n http $right($1,-5) SAVED %bvarout } } alias http.open { var %hashtable ; If two parameters are given, $1 is the name, else create a new name if ($0 > 1) { %hashtable = http. $+ $1 } else { var %h = $calc($ticks % $rand(1,1000000)) var %hashtable = http. $+ %h } if ($hget(%hashtable)) { echo -atg * /http.open: HTTP Request $qt($1) already exists return } ; Encode the octects of the URL. e.g.: foo bar,pie --> foo%20bar,%2Cpie var %url = $iif($0 > 1, $2-, $1-) var %urlEncoded = $http.urlencode(%url) ; Check the URL if (!$http.urlparse(%urlEncoded)) { echo -atg * /http.open: Malformed URL. Use /http.open [name] <url> return } if ($http.urlparse(%urlEncoded).secure && !$sslready) { echo -atg * /http.open: URL is SSL but $!sslready = false. See http://www.mirc.com/ssl.html return } ; Everything checks out, let's store the relevent bits in a hashtable .hmake %hashtable 10 http.seturl %hashtable %url echo -qatg * /http.open: Opened HTTP request $qt($right(%hashtable,-5)) } alias -l http.seturl { if ($0 < 2) { echo -atg HTTP Internal error, seturl called with no url } var %hashtable = $1 if (!$hget(%hashtable)) { echo -atg HTTP Internal error, seturl called with invalid handle } var %urlEncoded = $http.urlencode($2-) .hadd %hashtable host $http.urlparse(%urlEncoded).host .hadd %hashtable port $http.urlparse(%urlEncoded).port .hadd %hashtable secure $http.urlparse(%urlEncoded).secure .hadd %hashtable path $http.urlparse(%urlEncoded).path .hadd %hashtable user-agent mIRC $version } alias http.list { var %i = 1 var %numFound = 0 while (%i <= $hget(0)) { var %table = $hget(%i) inc %i if (http.* iswm %table) { inc %numFound echo -atg * $right(%table,-5) $+ : Host: $hget(%table,host) Port: $hget(%table,port) Path: $hget(%table,path) } } if (%numFound == 0) { echo -atg * No HTTP handles opened } } alias http.close { var %i = 1 var %numFound = 0 while (%i <= $hget(0)) { var %table = $hget(%i) if (http. $+ $1 iswm %table) { ; Cleanup resources if ($sock(%table)) { .sockclose %table } if ($fopen(%table)) { .fclose %table } if ($hget(%table $+ .postdata)) { .hfree %table $+ .postdata } .hfree %table inc %numFound echo -qatg * HTTP Closed $right(%table,-5) } ; Only inc if we didnt find a match else { inc %i } } if (%numFound == 0) { echo -qatg * No matching HTTP handles } } ; ; Takes an encoded URL and returns based on the property ; .secure - true if https ; .host - the URL host ; .port - the port, 80 by default ; .path - The path to download, / by default ; Returns $false is URL is malformed ; alias http.urlparse { var %secure var %host var %port var %path var %pathIndex var %protocol var %regex = /^(https?://)?([a-z.0-9\-_]+)(:\d+)?(/.*)?$/i if ($regex($1, %regex)) { if (http*:// iswm $regml(1)) { %protocol = $lower($left($regml(1),-3)) } %secure = $false if (%protocol == https) { %secure = $true } if (%protocol) { %host = $regml(2) } else { %host = $regml(1) } var %portIndex = $iif(%protocol, 3, 2) var %portfound = $false if ($left($regml(%portIndex),1) == : && $right($regml(%portIndex),-1) isnum) { %port = $right($regml(%portIndex),-1) %portFound = $true } else { if (%secure) { %port = 443 } else { %port = 80 } } %pathIndex = $iif(%portFound, $calc(%portIndex + 1), %portIndex) if ($regml(0) >= %pathIndex) { %path = $regml(%pathIndex) } else { %path = / } } else { return $false } if ($prop == secure) { return %secure } if ($prop == host) { return %host } if ($prop == path) { return %path } if ($prop == port) { return %port } return $true } alias http.addpost { if ($0 < 3) { echo -atg * /http.addpost: Usage: /http.addpost <handle> <variable> <value> return } var %maintable = http. $+ $1 var %posttable = %maintable $+ .postdata if (!$hget(%maintable)) { echo -atg * /http.addpost: No such HTTP handle found return } if (!$hget(%posttable)) { .hmake %posttable 10 } hadd %posttable $2 $3- echo -qatg Post variable added: $2 $+ = $+ $3- } alias http.delpost { if ($0 < 2) { echo -atg * /http.delpost: Usage: /http.delpost <handle> <variable> return } var %maintable = http. $+ $1 var %posttable = %maintable $+ .postdata if (!$hget(%maintable)) { echo -atg * /http.addpost: No such HTTP handle found return } if (!$hget(%posttable,$2)) { echo -qatg * /http.addpost: No post variable with that name exists return } hdel %posttable $2 echo -qatg Post variable deleted: $2 } ; Usage: $http.postdata(handle) ; Returns a binvar with a post data style query string (x=y&z=a) or $false alias http.postdata { ; Post data is stored in INNAME.postdata var %posttable = http. $+ $1 $+ .postdata if (!$hget(%posttable,0).item) { return $false } var %i = 1 while ($hget(%posttable,%i).item) { var %item = $ifmatch bset -t &bvPost $calc($bvar(&bvPost,0) +1) $iif(%i > 1,&,) $+ $http.urlencode(%item) $+ = $+ $http.urlencode($hget(%posttable,%item)) inc %i } return &bvPost } ; Replace some commonly touchy URL characters with their hex-octect encoding alias -l http.urlencode { return $replacex($1-,$chr(32),% $+ 20, $chr(44), % $+ 2C,+, % $+ 2B, $chr(37), % $+ 25) }
Contributed by aca20031 |