Manual is almost done. HTTP is missing.
Implemented new distribution scheme. Select is now purely C. HTTP reimplemented seems faster dunno why. LTN12 functions that coroutines fail gracefully.
This commit is contained in:
Diego Nehab
parent
9ed7f955e5
commit
58096449c6
40 changed files with 2035 additions and 815 deletions
357
src/http.lua
357
src/http.lua
|
|
@ -7,7 +7,7 @@
|
|||
-----------------------------------------------------------------------------
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
-- Load other required modules
|
||||
-- Load required modules
|
||||
-------------------------------------------------------------------------------
|
||||
local socket = require("socket")
|
||||
local ltn12 = require("ltn12")
|
||||
|
|
@ -17,42 +17,68 @@ local url = require("url")
|
|||
-----------------------------------------------------------------------------
|
||||
-- Setup namespace
|
||||
-------------------------------------------------------------------------------
|
||||
http = {}
|
||||
-- make all module globals fall into namespace
|
||||
setmetatable(http, { __index = _G })
|
||||
setfenv(1, http)
|
||||
_LOADED["http"] = getfenv(1)
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
-- Program constants
|
||||
-----------------------------------------------------------------------------
|
||||
-- connection timeout in seconds
|
||||
TIMEOUT = 60
|
||||
TIMEOUT = 4
|
||||
-- default port for document retrieval
|
||||
PORT = 80
|
||||
-- user agent field sent in request
|
||||
USERAGENT = socket.version
|
||||
USERAGENT = socket.VERSION
|
||||
-- block size used in transfers
|
||||
BLOCKSIZE = 2048
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
-- Function return value selectors
|
||||
-- Low level HTTP API
|
||||
-----------------------------------------------------------------------------
|
||||
local function second(a, b)
|
||||
return b
|
||||
local metat = { __index = {} }
|
||||
|
||||
function open(host, port)
|
||||
local con = socket.try(socket.tcp())
|
||||
socket.try(con:settimeout(TIMEOUT))
|
||||
port = port or PORT
|
||||
socket.try(con:connect(host, port))
|
||||
return setmetatable({ con = con }, metat)
|
||||
end
|
||||
|
||||
local function third(a, b, c)
|
||||
return c
|
||||
function metat.__index:sendrequestline(method, uri)
|
||||
local reqline = string.format("%s %s HTTP/1.1\r\n", method or "GET", uri)
|
||||
return socket.try(self.con:send(reqline))
|
||||
end
|
||||
|
||||
local function receive_headers(reqt, respt, tmp)
|
||||
local sock = tmp.sock
|
||||
function metat.__index:sendheaders(headers)
|
||||
for i, v in pairs(headers) do
|
||||
socket.try(self.con:send(i .. ": " .. v .. "\r\n"))
|
||||
end
|
||||
-- mark end of request headers
|
||||
socket.try(self.con:send("\r\n"))
|
||||
return 1
|
||||
end
|
||||
|
||||
function metat.__index:sendbody(headers, source, step)
|
||||
source = source or ltn12.source.empty()
|
||||
step = step or ltn12.pump.step
|
||||
-- if we don't know the size in advance, send chunked and hope for the best
|
||||
local mode
|
||||
if headers["content-length"] then mode = "keep-open"
|
||||
else mode = "http-chunked" end
|
||||
return socket.try(ltn12.pump.all(source, socket.sink(mode, self.con), step))
|
||||
end
|
||||
|
||||
function metat.__index:receivestatusline()
|
||||
local status = socket.try(self.con:receive())
|
||||
local code = socket.skip(2, string.find(status, "HTTP/%d*%.%d* (%d%d%d)"))
|
||||
return socket.try(tonumber(code), status)
|
||||
end
|
||||
|
||||
function metat.__index:receiveheaders()
|
||||
local line, name, value
|
||||
local headers = {}
|
||||
-- store results
|
||||
respt.headers = headers
|
||||
-- get first line
|
||||
line = socket.try(sock:receive())
|
||||
line = socket.try(self.con:receive())
|
||||
-- headers go until a blank line is found
|
||||
while line ~= "" do
|
||||
-- get field-name and value
|
||||
|
|
@ -60,189 +86,137 @@ local function receive_headers(reqt, respt, tmp)
|
|||
socket.try(name and value, "malformed reponse headers")
|
||||
name = string.lower(name)
|
||||
-- get next line (value might be folded)
|
||||
line = socket.try(sock:receive())
|
||||
line = socket.try(self.con:receive())
|
||||
-- unfold any folded values
|
||||
while string.find(line, "^%s") do
|
||||
value = value .. line
|
||||
line = socket.try(sock:receive())
|
||||
line = socket.try(self.con:receive())
|
||||
end
|
||||
-- save pair in table
|
||||
if headers[name] then headers[name] = headers[name] .. ", " .. value
|
||||
else headers[name] = value end
|
||||
end
|
||||
return headers
|
||||
end
|
||||
|
||||
local function receive_body(reqt, respt, tmp)
|
||||
local sink = reqt.sink or ltn12.sink.null()
|
||||
local step = reqt.step or ltn12.pump.step
|
||||
local source
|
||||
local te = respt.headers["transfer-encoding"]
|
||||
if te and te ~= "identity" then
|
||||
-- get by chunked transfer-coding of message body
|
||||
source = socket.source("http-chunked", tmp.sock)
|
||||
elseif tonumber(respt.headers["content-length"]) then
|
||||
-- get by content-length
|
||||
local length = tonumber(respt.headers["content-length"])
|
||||
source = socket.source("by-length", tmp.sock, length)
|
||||
else
|
||||
-- get it all until connection closes
|
||||
source = socket.source(tmp.sock)
|
||||
end
|
||||
socket.try(ltn12.pump.all(source, sink, step))
|
||||
function metat.__index:receivebody(headers, sink, step)
|
||||
sink = sink or ltn12.sink.null()
|
||||
step = step or ltn12.pump.step
|
||||
local length = tonumber(headers["content-length"])
|
||||
local TE = headers["transfer-encoding"]
|
||||
local mode
|
||||
if TE and TE ~= "identity" then mode = "http-chunked"
|
||||
elseif tonumber(headers["content-length"]) then mode = "by-length"
|
||||
else mode = "default" end
|
||||
return socket.try(ltn12.pump.all(socket.source(mode, self.con, length),
|
||||
sink, step))
|
||||
end
|
||||
|
||||
local function send_headers(sock, headers)
|
||||
-- send request headers
|
||||
for i, v in pairs(headers) do
|
||||
socket.try(sock:send(i .. ": " .. v .. "\r\n"))
|
||||
end
|
||||
-- mark end of request headers
|
||||
socket.try(sock:send("\r\n"))
|
||||
function metat.__index:close()
|
||||
return self.con:close()
|
||||
end
|
||||
|
||||
local function should_receive_body(reqt, respt, tmp)
|
||||
if reqt.method == "HEAD" then return nil end
|
||||
if respt.code == 204 or respt.code == 304 then return nil end
|
||||
if respt.code >= 100 and respt.code < 200 then return nil end
|
||||
return 1
|
||||
end
|
||||
|
||||
local function receive_status(reqt, respt, tmp)
|
||||
local status = socket.try(tmp.sock:receive())
|
||||
local code = third(string.find(status, "HTTP/%d*%.%d* (%d%d%d)"))
|
||||
-- store results
|
||||
respt.code, respt.status = tonumber(code), status
|
||||
end
|
||||
|
||||
local function request_uri(reqt, respt, tmp)
|
||||
local u = tmp.parsed
|
||||
if not reqt.proxy then
|
||||
local parsed = tmp.parsed
|
||||
u = {
|
||||
path = parsed.path,
|
||||
params = parsed.params,
|
||||
query = parsed.query,
|
||||
fragment = parsed.fragment
|
||||
-----------------------------------------------------------------------------
|
||||
-- High level HTTP API
|
||||
-----------------------------------------------------------------------------
|
||||
local function uri(reqt)
|
||||
local u = reqt
|
||||
if not reqt.proxy and not PROXY then
|
||||
u = {
|
||||
path = reqt.path,
|
||||
params = reqt.params,
|
||||
query = reqt.query,
|
||||
fragment = reqt.fragment
|
||||
}
|
||||
end
|
||||
return url.build(u)
|
||||
end
|
||||
|
||||
local function send_request(reqt, respt, tmp)
|
||||
local uri = request_uri(reqt, respt, tmp)
|
||||
local headers = tmp.headers
|
||||
local step = reqt.step or ltn12.pump.step
|
||||
-- send request line
|
||||
socket.try(tmp.sock:send((reqt.method or "GET")
|
||||
.. " " .. uri .. " HTTP/1.1\r\n"))
|
||||
if reqt.source and not headers["content-length"] then
|
||||
headers["transfer-encoding"] = "chunked"
|
||||
end
|
||||
send_headers(tmp.sock, headers)
|
||||
-- send request message body, if any
|
||||
if not reqt.source then return end
|
||||
if headers["content-length"] then
|
||||
socket.try(ltn12.pump.all(reqt.source,
|
||||
socket.sink(tmp.sock), step))
|
||||
else
|
||||
socket.try(ltn12.pump.all(reqt.source,
|
||||
socket.sink("http-chunked", tmp.sock), step))
|
||||
end
|
||||
end
|
||||
|
||||
local function open(reqt, respt, tmp)
|
||||
local proxy = reqt.proxy or PROXY
|
||||
local host, port
|
||||
if proxy then
|
||||
local pproxy = url.parse(proxy)
|
||||
socket.try(pproxy.port and pproxy.host, "invalid proxy")
|
||||
host, port = pproxy.host, pproxy.port
|
||||
else
|
||||
host, port = tmp.parsed.host, tmp.parsed.port
|
||||
end
|
||||
-- store results
|
||||
tmp.sock = socket.try(socket.tcp())
|
||||
socket.try(tmp.sock:settimeout(reqt.timeout or TIMEOUT))
|
||||
socket.try(tmp.sock:connect(host, port))
|
||||
end
|
||||
|
||||
local function adjust_headers(reqt, respt, tmp)
|
||||
local function adjustheaders(headers, host)
|
||||
local lower = {}
|
||||
-- override with user values
|
||||
for i,v in (reqt.headers or lower) do
|
||||
for i,v in (headers or lower) do
|
||||
lower[string.lower(i)] = v
|
||||
end
|
||||
lower["user-agent"] = lower["user-agent"] or USERAGENT
|
||||
-- these cannot be overriden
|
||||
lower["host"] = tmp.parsed.host
|
||||
lower["connection"] = "close"
|
||||
-- store results
|
||||
tmp.headers = lower
|
||||
lower["host"] = host
|
||||
return lower
|
||||
end
|
||||
|
||||
local function parse_url(reqt, respt, tmp)
|
||||
local function adjustrequest(reqt)
|
||||
-- parse url with default fields
|
||||
local parsed = url.parse(reqt.url, {
|
||||
host = "",
|
||||
port = PORT,
|
||||
port = PORT,
|
||||
path ="/",
|
||||
scheme = "http"
|
||||
scheme = "http"
|
||||
})
|
||||
-- scheme has to be http
|
||||
socket.try(parsed.scheme == "http",
|
||||
string.format("unknown scheme '%s'", parsed.scheme))
|
||||
-- explicit authentication info overrides that given by the URL
|
||||
parsed.user = reqt.user or parsed.user
|
||||
parsed.password = reqt.password or parsed.password
|
||||
-- store results
|
||||
tmp.parsed = parsed
|
||||
-- explicit info in reqt overrides that given by the URL
|
||||
for i,v in reqt do parsed[i] = v end
|
||||
-- compute uri if user hasn't overriden
|
||||
parsed.uri = parsed.uri or uri(parsed)
|
||||
-- adjust headers in request
|
||||
parsed.headers = adjustheaders(parsed.headers, parsed.host)
|
||||
return parsed
|
||||
end
|
||||
|
||||
-- forward declaration
|
||||
local request_p
|
||||
local function shouldredirect(reqt, respt)
|
||||
return (reqt.redirect ~= false) and
|
||||
(respt.code == 301 or respt.code == 302) and
|
||||
(not reqt.method or reqt.method == "GET" or reqt.method == "HEAD")
|
||||
and (not reqt.nredirects or reqt.nredirects < 5)
|
||||
end
|
||||
|
||||
local function should_authorize(reqt, respt, tmp)
|
||||
local function shouldauthorize(reqt, respt)
|
||||
-- if there has been an authorization attempt, it must have failed
|
||||
if reqt.headers and reqt.headers["authorization"] then return nil end
|
||||
-- if last attempt didn't fail due to lack of authentication,
|
||||
-- or we don't have authorization information, we can't retry
|
||||
return respt.code == 401 and tmp.parsed.user and tmp.parsed.password
|
||||
return respt.code == 401 and reqt.user and reqt.password
|
||||
end
|
||||
|
||||
local function clone(headers)
|
||||
if not headers then return nil end
|
||||
local copy = {}
|
||||
for i,v in pairs(headers) do
|
||||
copy[i] = v
|
||||
local function shouldreceivebody(reqt, respt)
|
||||
if reqt.method == "HEAD" then return nil end
|
||||
local code = respt.code
|
||||
if code == 204 or code == 304 then return nil end
|
||||
if code >= 100 and code < 200 then return nil end
|
||||
return 1
|
||||
end
|
||||
|
||||
local requestp, authorizep, redirectp
|
||||
|
||||
function requestp(reqt)
|
||||
local reqt = adjustrequest(reqt)
|
||||
local respt = {}
|
||||
local con = open(reqt.host, reqt.port)
|
||||
con:sendrequestline(reqt.method, reqt.uri)
|
||||
con:sendheaders(reqt.headers)
|
||||
con:sendbody(reqt.headers, reqt.source, reqt.step)
|
||||
respt.code, respt.status = con:receivestatusline()
|
||||
respt.headers = con:receiveheaders()
|
||||
if shouldredirect(reqt, respt) then
|
||||
con:close()
|
||||
return redirectp(reqt, respt)
|
||||
elseif shouldauthorize(reqt, respt) then
|
||||
con:close()
|
||||
return authorizep(reqt, respt)
|
||||
elseif shouldreceivebody(reqt, respt) then
|
||||
con:receivebody(respt.headers, reqt.sink, reqt.step)
|
||||
end
|
||||
return copy
|
||||
con:close()
|
||||
return respt
|
||||
end
|
||||
|
||||
local function authorize(reqt, respt, tmp)
|
||||
local headers = clone(reqt.headers) or {}
|
||||
headers["authorization"] = "Basic " ..
|
||||
(mime.b64(tmp.parsed.user .. ":" .. tmp.parsed.password))
|
||||
local autht = {
|
||||
method = reqt.method,
|
||||
url = reqt.url,
|
||||
source = reqt.source,
|
||||
sink = reqt.sink,
|
||||
headers = headers,
|
||||
timeout = reqt.timeout,
|
||||
proxy = reqt.proxy,
|
||||
}
|
||||
request_p(autht, respt, tmp)
|
||||
function authorizep(reqt, respt)
|
||||
local auth = "Basic " .. (mime.b64(reqt.user .. ":" .. reqt.password))
|
||||
reqt.headers["authorization"] = auth
|
||||
return requestp(reqt)
|
||||
end
|
||||
|
||||
local function should_redirect(reqt, respt, tmp)
|
||||
return (reqt.redirect ~= false) and
|
||||
(respt.code == 301 or respt.code == 302) and
|
||||
(not reqt.method or reqt.method == "GET" or reqt.method == "HEAD")
|
||||
and (not tmp.nredirects or tmp.nredirects < 5)
|
||||
end
|
||||
|
||||
local function redirect(reqt, respt, tmp)
|
||||
tmp.nredirects = (tmp.nredirects or 0) + 1
|
||||
function redirectp(reqt, respt)
|
||||
-- we create a new table to get rid of anything we don't
|
||||
-- absolutely need, including authentication info
|
||||
local redirt = {
|
||||
method = reqt.method,
|
||||
-- the RFC says the redirect URL has to be absolute, but some
|
||||
|
|
@ -251,69 +225,38 @@ local function redirect(reqt, respt, tmp)
|
|||
source = reqt.source,
|
||||
sink = reqt.sink,
|
||||
headers = reqt.headers,
|
||||
timeout = reqt.timeout,
|
||||
proxy = reqt.proxy
|
||||
proxy = reqt.proxy,
|
||||
nredirects = (reqt.nredirects or 0) + 1
|
||||
}
|
||||
request_p(redirt, respt, tmp)
|
||||
respt = requestp(redirt)
|
||||
-- we pass the location header as a clue we redirected
|
||||
if respt.headers then respt.headers.location = redirt.url end
|
||||
end
|
||||
|
||||
local function skip_continue(reqt, respt, tmp)
|
||||
if respt.code == 100 then
|
||||
receive_status(reqt, respt, tmp)
|
||||
end
|
||||
end
|
||||
|
||||
-- execute a request of through an exception
|
||||
function request_p(reqt, respt, tmp)
|
||||
parse_url(reqt, respt, tmp)
|
||||
adjust_headers(reqt, respt, tmp)
|
||||
open(reqt, respt, tmp)
|
||||
send_request(reqt, respt, tmp)
|
||||
receive_status(reqt, respt, tmp)
|
||||
skip_continue(reqt, respt, tmp)
|
||||
receive_headers(reqt, respt, tmp)
|
||||
if should_redirect(reqt, respt, tmp) then
|
||||
tmp.sock:close()
|
||||
redirect(reqt, respt, tmp)
|
||||
elseif should_authorize(reqt, respt, tmp) then
|
||||
tmp.sock:close()
|
||||
authorize(reqt, respt, tmp)
|
||||
elseif should_receive_body(reqt, respt, tmp) then
|
||||
receive_body(reqt, respt, tmp)
|
||||
end
|
||||
end
|
||||
|
||||
function request(reqt)
|
||||
local respt, tmp = {}, {}
|
||||
local s, e = pcall(request_p, reqt, respt, tmp)
|
||||
if not s then respt.error = e end
|
||||
if tmp.sock then tmp.sock:close() end
|
||||
return respt
|
||||
end
|
||||
|
||||
function get(u)
|
||||
local t = {}
|
||||
respt = request {
|
||||
url = u,
|
||||
sink = ltn12.sink.table(t)
|
||||
}
|
||||
return (table.getn(t) > 0 or nil) and table.concat(t), respt.headers,
|
||||
respt.code, respt.error
|
||||
end
|
||||
request = socket.protect(requestp)
|
||||
|
||||
function post(u, body)
|
||||
get = socket.protect(function(u)
|
||||
local t = {}
|
||||
respt = request {
|
||||
url = u,
|
||||
method = "POST",
|
||||
local respt = requestp {
|
||||
url = u,
|
||||
sink = ltn12.sink.table(t)
|
||||
}
|
||||
return (table.getn(t) > 0 or nil) and table.concat(t), respt.headers,
|
||||
respt.code
|
||||
end)
|
||||
|
||||
post = socket.protect(function(u, body)
|
||||
local t = {}
|
||||
local respt = requestp {
|
||||
url = u,
|
||||
method = "POST",
|
||||
source = ltn12.source.string(body),
|
||||
sink = ltn12.sink.table(t),
|
||||
headers = { ["content-length"] = string.len(body) }
|
||||
headers = { ["content-length"] = string.len(body) }
|
||||
}
|
||||
return (table.getn(t) > 0 or nil) and table.concat(t),
|
||||
respt.headers, respt.code, respt.error
|
||||
end
|
||||
return (table.getn(t) > 0 or nil) and table.concat(t),
|
||||
respt.headers, respt.code
|
||||
end)
|
||||
|
||||
return http
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue