split form upload and wx upload, refactor file structure

README.md

@@ -6,7 +6,7 @@
 ➤ ./cp_scripts_local_macos.sh                                                                                                                                                                                                                                                                                   20:31:54
 + mkdir -p /usr/local/etc/openresty/conf.d
 + cp conf/local_macos/admin.conf conf/local_macos/music-room.conf conf/local_macos/ts.conf /usr/local/etc/openresty/conf.d/
-+ cp scripts/combine.lua scripts/convert_audio.lua scripts/strip_path.lua scripts/test_upyun_token.lua scripts/test_upyun_upload.lua scripts/upload.lua scripts/upload_audio.lua scripts/upyun.lua scripts/upyun_token.lua /usr/local/opt/openresty/resty_funcs/
++ cp scripts/*.lua /usr/local/opt/openresty/resty_funcs/
 ```
 
 第一次更新记得在`nginx.conf`里面include配置目录`conf.d`:

conf/alchemy/admin.conf

@@ -51,10 +51,21 @@ server {
     location /api/ngx/image/upload_combined {
          content_by_lua_file $resty_loc/nginx/scripts/upload_combined_image.lua;
     }
-
     #Upload image files
     location /api/ngx/image/upload {
-         content_by_lua_file $resty_loc/nginx/scripts/upload.lua;
+         content_by_lua_file $resty_loc/nginx/scripts/old_upload.lua;
+    }
+    #upload image from weixin media
+    location /api/ngx/image/wx_upload_single {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_single.lua;
+    }
+    #upload image files to combine by form
+    location /api/ngx/image/form_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/form_upload_to_combine.lua;
+    }
+    #Upload image files to combine by wx
+    location /api/ngx/image/wx_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_to_combine.lua;
     }
     #Combine image files
     location /api/ngx/image/combine {

conf/local_linux/admin.conf

@@ -50,7 +50,19 @@ server {
 
     #Upload image files
     location /api/ngx/image/upload {
-         content_by_lua_file $resty_loc/nginx/scripts/upload.lua;
+         content_by_lua_file $resty_loc/nginx/scripts/old_upload.lua;
+    }
+    #upload image from weixin media
+    location /api/ngx/image/wx_upload_single {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_single.lua;
+    }
+    #upload image files to combine by form
+    location /api/ngx/image/form_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/form_upload_to_combine.lua;
+    }
+    #Upload image files to combine by wx
+    location /api/ngx/image/wx_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_to_combine.lua;
     }
     #Combine image files
     location /api/ngx/image/combine {

conf/local_macos/admin.conf

@@ -41,7 +41,19 @@ server {
 
     #Upload image files
     location /api/ngx/image/upload {
-         content_by_lua_file $resty_loc/nginx/scripts/upload.lua;
+         content_by_lua_file $resty_loc/nginx/scripts/old_upload.lua;
+    }
+    #upload image from weixin media
+    location /api/ngx/image/wx_upload_single {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_single.lua;
+    }
+    #upload image files to combine by form
+    location /api/ngx/image/form_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/form_upload_to_combine.lua;
+    }
+    #Upload image files to combine by wx
+    location /api/ngx/image/wx_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_to_combine.lua;
     }
 
     #Combine image files

conf/moicen/admin.conf

@@ -48,7 +48,19 @@ server {
 
     #Upload image files
     location /api/ngx/image/upload {
-         content_by_lua_file $resty_loc/nginx/scripts/upload.lua;
+         content_by_lua_file $resty_loc/nginx/scripts/old_upload.lua;
+    }
+    #upload image from weixin media
+    location /api/ngx/image/wx_upload_single {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_single.lua;
+    }
+    #upload image files to combine by form
+    location /api/ngx/image/form_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/form_upload_to_combine.lua;
+    }
+    #Upload image files to combine by wx
+    location /api/ngx/image/wx_upload_to_combine {
+         content_by_lua_file $resty_loc/nginx/scripts/wx_upload_to_combine.lua;
     }
     #Combine image files
     location /api/ngx/image/combine {

cp_scripts_alchemy.sh

@@ -24,10 +24,7 @@ mkdir -p $openresty_lua_scripts_dir
 cp scripts/*.lua $openresty_lua_scripts_dir
 # -- Replace scripts path in file
 sed -i -e "s|<SCRIPT_PATH>|$openresty_lua_scripts_dir|" \
-  "$openresty_lua_scripts_dir"/combine.lua \
+  "$openresty_lua_scripts_dir"/*.lua
-  "$openresty_lua_scripts_dir"/convert_audio.lua \
-  "$openresty_lua_scripts_dir"/test_upyun_upload.lua \
-  "$openresty_lua_scripts_dir"/upyun_upload.lua
 
 # Reload openresty
 openresty -t

cp_scripts_moicen.sh

@@ -24,10 +24,7 @@ mkdir -p $openresty_lua_scripts_dir
 cp scripts/*.lua $openresty_lua_scripts_dir
 # -- Replace scripts path in file
 sed -i -e "s|<SCRIPT_PATH>|$openresty_lua_scripts_dir|" \
-  "$openresty_lua_scripts_dir"/combine.lua \
+  "$openresty_lua_scripts_dir"/*.lua
-  "$openresty_lua_scripts_dir"/convert_audio.lua \
-  "$openresty_lua_scripts_dir"/test_upyun_upload.lua \
-  "$openresty_lua_scripts_dir"/upyun_upload.lua
 
 # Reload openresty
 openresty -t

scripts/convert_audio.lua

@@ -4,95 +4,8 @@ package.path = package.path .. ';<SCRIPT_PATH>/?.lua';
 
 local cjson = require "cjson"
 local uuid = require "resty.jit-uuid"
-local http = require "resty.http"
 local upyun_upload = require("upyun_upload")
-
+local get_wx_media = require("wx_download")
-local TaskTypes = {
-    NOOP = 'NOOP',
-    CONVERT_AUDIO_FILE = "CONVERT_AUDIO_FILE"
-}
-
-local httpc = http.new()
-local sudoerToken = ngx.req.get_headers().HtySudoerToken
-local htyhostHeader = ngx.req.get_headers().HtyHost
-
-local function get_access_token()
-
-    local htyuc = ngx.var.htyuc
-    local host = ngx.var.host
-    local wx_domain = ngx.var.wx_domain
-
-    ngx.log(ngx.INFO, "Request URL -> ", ngx.var.uri)
-    ngx.log(ngx.INFO, "HTYUC -> ", htyuc)
-    ngx.log(ngx.INFO, "Host -> ", host)
-    ngx.log(ngx.INFO, 'HtyHostHeader -> ', htyhostHeader)
-    ngx.log(ngx.INFO, 'HtySudoerToken -> ', sudoerToken)
-
-    local remote_url = string.format("%s/api/v1/uc/wx_get_access_token", htyuc)
-    ngx.log(ngx.INFO, 'remote_url -> ', remote_url)
-
-    local res, err = httpc:request_uri(
-            remote_url,
-            {
-                ssl_verify = false, -- 设置参数 ssl_verify 为false 不校验ssl证书
-                method = "GET",
-                headers = {
-                    ["HtyHost"] = wx_domain,
-                    ["HtySudoerToken"] = sudoerToken,
-                }
-            }
-    )
-
-    if res == nil then
-        ngx.log(ngx.ERR, "FAILED TO CONNECT TO *HTYUC*", err)
-    end
-
-    if 200 ~= res.status then
-        ngx.log(ngx.ERR, "GET ACCESS TOKEN *FAILED*", err)
-        ngx.say(err)
-        ngx.exit(res.status)
-    else
-        ngx.log(ngx.ERR, res.body)
-        return cjson.decode(res.body).d
-    end
-end
-
-local function get_wx_media(media_id)
-    local access_token = get_access_token()
-    print("access token: " .. access_token)
-    local wx_media_url = string.format("https://api.weixin.qq.com/cgi-bin/media/get?access_token=%s&media_id=%s", access_token, media_id)
-    print("wx_media_url: " .. wx_media_url)
-    local res, err = httpc:request_uri(wx_media_url, {
-        method = 'GET',
-        ssl_verify = ssl_verify or false, -- 设置参数 ssl_verify 为false 不校验ssl证书
-    })
-
-    if res == nil then
-        ngx.log(ngx.ERR, "FAILED TO CONNECT TO *Weixin*", err)
-        ngx.say(err)
-        ngx.exit(500)
-    end
-
-    if 200 ~= res.status then
-        ngx.log(ngx.ERR, 'GET WECHET MEDIA *FAILED*', err)
-        ngx.say(err)
-        ngx.exit(res.status)
-    end
-
-    local filename = ngx.re.match(res.headers['Content-disposition'], [[filename="(.+\.(amr|speex))"]], "jo")[1]
-    local file_dir = ngx.var.tmp_file_dir
-    local saved_audio = file_dir .. "/" .. filename;
-    print('saved audio file...' .. saved_audio);
-    local file, err = io.open(saved_audio, 'w')
-    if file == nil then
-        print("Can not open file..." .. err)
-    else
-        file:write(res.body)
-        file:close()
-    end
-
-    return saved_audio;
-end
 
 local function convert()
 

scripts/create_task.lua

@@ -0,0 +1,54 @@
+local cjson = require "cjson"
+local http = require "resty.http"
+
+
+local TaskTypes = {
+    NOOP = 'NOOP',
+    UPLOAD_PICTURE = 'UPLOAD_PICTURE'
+}
+
+local task_server = ngx.var.task_server
+
+local htyhost = ngx.req.get_headers().HtyHost
+
+ngx.log(ngx.INFO, "TASK_SERVER -> ", task_server)
+ngx.log(ngx.INFO, "HTY HOST -> ", htyhost)
+
+
+local function create_task(files, authHeader, sudoerToken)
+    local httpc = http.new()
+    local remote_url = string.format("%s/api/v1/ts/create_task", task_server)
+    ngx.log(ngx.INFO, 'remote_url -> ', remote_url)
+    ngx.log(ngx.INFO, 'Authorization -> ', authHeader)
+    ngx.log(ngx.INFO, 'HtySudoerToken -> ', sudoerToken)
+
+    local body_text = cjson.encode({ task_type = TaskTypes.UPLOAD_PICTURE, payload = { images = files } })
+
+    ngx.log(ngx.INFO, 'UPLOAD_PICTURE *body_text* ->', body_text)
+    local res, err = httpc:request_uri(
+            remote_url,
+            {
+                ssl_verify = false, -- 设置参数 ssl_verify 为false 不校验ssl证书
+                method = "POST",
+                headers = {
+                    ["Content-Type"] = "application/json",
+                    ["Authorization"] = authHeader,
+                    ["HtySudoerToken"] = sudoerToken,
+                    ['HtyHost'] = htyhost
+                },
+                body = body_text,
+            }
+    )
+    if res == nil then
+        ngx.log(ngx.ERR, "FAILED TO CONNECT TO *TASK_SERVER*", err)
+    end
+
+    if 201 ~= res.status then
+        ngx.log(ngx.ERR, "TASK CREATE *FAILED*", res.body)
+
+    end
+    ngx.say(res.body)
+    ngx.exit(res.status)
+end
+
+return create_task

scripts/form_upload_to_combine.lua

@@ -0,0 +1,81 @@
+-- 指定模块引用目录，否则无法加载同目录下的其他文件
+--package.path = package.path .. ';/usr/local/opt/openresty/nginx/scripts/?.lua';
+package.path = package.path .. ';<SCRIPT_PATH>/?.lua';
+
+local upload = require "resty.upload"
+local uuid = require "resty.jit-uuid"
+local verify = require('jwt_verify')
+local create_task = require("create_task")
+
+local authHeader = ngx.req.get_headers().Authorization
+local sudoerToken = ngx.req.get_headers().HtySudoerToken
+
+verify(authHeader, sudoerToken)
+
+local function read_form_file()
+    local chunk_size = 4096
+    local form, err = upload:new(chunk_size)
+    if not form then
+        ngx.log(ngx.ERR, "failed to new upload: ", err)
+        ngx.exit(500)
+    end
+    uuid.seed()
+
+    local file_dir = ngx.var.tmp_file_dir
+    local file
+    local file_name
+    local files = {}
+
+    while true do
+        local type, res, err = form:read()
+
+        if not type then
+            ngx.say("FAILED TO READ *UPLOAD IMAGE* -> ", err)
+            return
+        end
+
+        if type == "header" then
+            --"Content-Disposition","form-data; name=\"files[]\"; filename=\"Song-of-joy.png\""
+            --"Content-Type","image\/png"
+            local key = res[1]
+            local val = res[2]
+            if key == "Content-Type" then
+                local ext = ngx.re.match(val, [[(\w+)\/(\w+)]], "jo")[2]
+                file_name = uuid() .. "." .. ext
+            end
+
+            if file_name then
+                file = io.open(file_dir .. "/" .. file_name, "w+")
+                ngx.log(ngx.INFO, "FILENAME -> ", file_name)
+                if not file then
+                    ngx.say("failed to open file ", file_name)
+                    return
+                end
+            end
+
+        elseif type == "body" then
+            if file then
+                file:write(res)
+                -- sha1:update(res)
+            end
+        elseif type == "part_end" then
+            if file then
+                file:close()
+                table.insert(files, file_name)
+            end
+            -- 这里要重置一下file_name，否则后面的文件保存时会导致前面已保存的文件变成空文件
+            -- file:flush() 和 io.flush() 都没效果
+            file_name = nil
+            file = nil
+        elseif type == "eof" then
+            create_task(files, authHeader, sudoerToken)
+            break
+        else
+            -- do nothing
+        end
+    end
+end
+
+
+read_form_file()
+

scripts/jwt_verify.lua

@@ -0,0 +1,104 @@
+local cjson = require "cjson"
+--# https://github.com/SkyLothar/lua-resty-jwt
+--$ opm get SkyLothar/lua-resty-jwt
+local jwt = require "resty.jwt"
+
+local function cleanupString(str, remove)
+    local lcSubStrTab = {}
+    while true do
+        local lcPos = string.find(str, remove)
+        if not lcPos then
+            lcSubStrTab[#lcSubStrTab + 1] = str
+            break
+        end
+        local lcSubStr = string.sub(str, 1, lcPos - 1)
+        lcSubStrTab[#lcSubStrTab + 1] = lcSubStr
+        str = string.sub(str, lcPos + 1, #str)
+    end
+    local lcMergeStr = ""
+    local lci = 1
+    while true do
+        if lcSubStrTab[lci] then
+            lcMergeStr = lcMergeStr .. lcSubStrTab[lci]
+            lci = lci + 1
+        else
+            break
+        end
+    end
+    return lcMergeStr
+end
+
+local function verifyJwtToken(token)
+    local htyuc = ngx.var.htyuc
+    local htyhost = ngx.req.get_headers().HtyHost
+
+    local http = require "resty.http"
+    local httpc = http.new()
+    local verify_jwt_url = string.format("%s/api/v1/uc/verify_jwt_token", htyuc)
+
+    ngx.log(ngx.INFO, 'HTYUC VERIFY_JWT_TOKEN_URL -> ' .. verify_jwt_url)
+
+    local res, err = httpc:request_uri(
+            verify_jwt_url,
+            {
+                ssl_verify = false, -- 设置参数 ssl_verify 为false 不校验ssl证书
+                method = "POST",
+                headers = {
+                    ['HtyHost'] = htyhost,
+                    ["Authorization"] = token,
+                },
+            }
+    )
+
+    if not res then
+        ngx.log(ngx.ERR, "CAN'T VERIFY JWT TOKEN -> ", err)
+        ngx.exit(ngx.HTTP_FORBIDDEN)
+    else
+        if 200 ~= res.status then
+            ngx.log(ngx.ERR, "JWT TOKEN VERIFICATION *ERROR* -> ", err)
+            ngx.say(res.body)
+            ngx.exit(res.status)
+        end
+    end
+
+    ngx.log(ngx.INFO, "JWT TOKEN VERIFICATION *PASSED*.")
+
+end
+
+local function decodeJwtClaim(jwtClaim)
+    local jwtKey = "0xCAFEBABE0xCAFEBABE0xCAFEBABE0xCAFEBABE0xCAFEBABE0xCAFEBABE"
+    local jwtObj = jwt:verify(jwtKey, jwtClaim)
+
+    if jwtObj.verified == false then
+        ngx.log(ngx.WARN, "INVALID TOKEN -> " .. jwtObj.reason)
+        ngx.status = ngx.HTTP_UNAUTHORIZED
+        ngx.header.content_type = "application/json; charset=utf-8"
+        ngx.say(cjson.encode(jwtObj))
+        ngx.exit(ngx.HTTP_UNAUTHORIZED)
+    end
+
+    ngx.log(ngx.INFO, "DECODED JWT CLAIM -> " .. cjson.encode(jwtObj))
+
+    local htyToken = cjson.decode(jwtObj.payload.sub)
+    ngx.log(ngx.INFO, "DECODED JWT TOKEN -> " .. cjson.encode(htyToken))
+    return htyToken
+end
+
+local function verify(authHeader, sudoerToken)
+
+    if authHeader then
+        ngx.log(ngx.INFO, 'CHECK_AUTH_HEADER -> ', authHeader)
+        --    local token = decodeJwtClaim(authHeader)
+        verifyJwtToken(authHeader)
+    elseif sudoerToken then
+        ngx.log(ngx.INFO, 'CHECK_SUDOER_HEADER -> ', sudoerToken)
+        --    local token = decodeJwtClaim(sudoerToken)
+        verifyJwtToken(sudoerToken)
+    else
+        ngx.log(ngx.ERR, 'NO AUTH/SUDOER HEADERS!')
+        ngx.status = 500
+        ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
+    end
+end
+
+return verify

scripts/old_upload.lua

@@ -113,11 +113,11 @@ local sudoerToken = ngx.req.get_headers().HtySudoerToken
 
 if authHeader then
     ngx.log(ngx.INFO, 'CHECK_AUTH_HEADER -> ', authHeader)
---    local token = decodeJwtClaim(authHeader)
+    --    local token = decodeJwtClaim(authHeader)
     verifyJwtToken(authHeader)
 elseif sudoerToken then
     ngx.log(ngx.INFO, 'CHECK_SUDOER_HEADER -> ', sudoerToken)
---    local token = decodeJwtClaim(sudoerToken)
+    --    local token = decodeJwtClaim(sudoerToken)
     verifyJwtToken(sudoerToken)
 else
     ngx.log(ngx.ERR, 'NO AUTH/SUDOER HEADERS!')

scripts/wx_download.lua

@@ -0,0 +1,87 @@
+local cjson = require "cjson"
+local http = require "resty.http"
+
+local httpc = http.new()
+local sudoerToken = ngx.req.get_headers().HtySudoerToken
+local htyhostHeader = ngx.req.get_headers().HtyHost
+
+
+local function get_access_token()
+
+    local htyuc = ngx.var.htyuc
+    local host = ngx.var.host
+    local wx_domain = ngx.var.wx_domain
+
+    ngx.log(ngx.INFO, "Request URL -> ", ngx.var.uri)
+    ngx.log(ngx.INFO, "HTYUC -> ", htyuc)
+    ngx.log(ngx.INFO, "Host -> ", host)
+    ngx.log(ngx.INFO, 'HtyHostHeader -> ', htyhostHeader)
+    ngx.log(ngx.INFO, 'HtySudoerToken -> ', sudoerToken)
+
+    local remote_url = string.format("%s/api/v1/uc/wx_get_access_token", htyuc)
+    ngx.log(ngx.INFO, 'remote_url -> ', remote_url)
+
+    local res, err = httpc:request_uri(
+            remote_url,
+            {
+                ssl_verify = false, -- 设置参数 ssl_verify 为false 不校验ssl证书
+                method = "GET",
+                headers = {
+                    ["HtyHost"] = wx_domain,
+                    ["HtySudoerToken"] = sudoerToken,
+                }
+            }
+    )
+
+    if res == nil then
+        ngx.log(ngx.ERR, "FAILED TO CONNECT TO *HTYUC*", err)
+    end
+
+    if 200 ~= res.status then
+        ngx.log(ngx.ERR, "GET ACCESS TOKEN *FAILED*", err)
+        ngx.say(err)
+        ngx.exit(res.status)
+    else
+        ngx.log(ngx.ERR, res.body)
+        return cjson.decode(res.body).d
+    end
+end
+
+local function get_wx_media(media_id)
+    local access_token = get_access_token()
+    print("access token: " .. access_token)
+    local wx_media_url = string.format("https://api.weixin.qq.com/cgi-bin/media/get?access_token=%s&media_id=%s", access_token, media_id)
+    print("wx_media_url: " .. wx_media_url)
+    local res, err = httpc:request_uri(wx_media_url, {
+        method = 'GET',
+        ssl_verify = ssl_verify or false, -- 设置参数 ssl_verify 为false 不校验ssl证书
+    })
+
+    if res == nil then
+        ngx.log(ngx.ERR, "FAILED TO CONNECT TO *Weixin*", err)
+        ngx.say(err)
+        ngx.exit(500)
+    end
+
+    if 200 ~= res.status then
+        ngx.log(ngx.ERR, 'GET WECHET MEDIA *FAILED*', err)
+        ngx.say(err)
+        ngx.exit(res.status)
+    end
+
+    local filename = ngx.re.match(res.headers['Content-disposition'], [[filename="(.+\.(amr|speex))"]], "jo")[1]
+    local file_dir = ngx.var.tmp_file_dir
+    local saved_file = file_dir .. "/" .. filename;
+    print('saved file...' .. saved_file);
+    local file, err = io.open(saved_file, 'w')
+    if file == nil then
+        print("Can not open file..." .. err)
+    else
+        file:write(res.body)
+        file:close()
+    end
+
+    return saved_file;
+end
+
+return get_wx_media;

scripts/wx_upload_single.lua

@@ -0,0 +1,26 @@
+-- 指定模块引用目录，否则无法加载同目录下的其他文件
+--package.path = package.path .. ';/usr/local/opt/openresty/nginx/scripts/?.lua';
+package.path = package.path .. ';<SCRIPT_PATH>/?.lua';
+
+local cjson = require "cjson"
+local upyun_upload = require("upyun_upload")
+local get_wx_media = require("wx_download")
+
+local verify = require('jwt_verify')
+
+local authHeader = ngx.req.get_headers().Authorization
+local sudoerToken = ngx.req.get_headers().HtySudoerToken
+verify(authHeader, sudoerToken)
+
+local function run()
+
+    ngx.req.read_body()
+    local req_body = cjson.decode(ngx.req.get_body_data())
+    ngx.log(ngx.INFO, 'REQ_BODY -> ', ngx.req.get_body_data())
+
+    local downloaded_file = get_wx_media(req_body["payload"]["media_id"]);
+
+    upyun_upload.upload(downloaded_file)
+end
+
+run();

scripts/wx_upload_to_combine.lua

@@ -0,0 +1,32 @@
+-- 指定模块引用目录，否则无法加载同目录下的其他文件
+--package.path = package.path .. ';/usr/local/opt/openresty/nginx/scripts/?.lua';
+package.path = package.path .. ';<SCRIPT_PATH>/?.lua';
+
+local cjson = require "cjson"
+local get_wx_media = require("wx_download")
+local create_task = require("create_task")
+local verify = require('jwt_verify')
+
+
+local authHeader = ngx.req.get_headers().Authorization
+local sudoerToken = ngx.req.get_headers().HtySudoerToken
+verify(authHeader, sudoerToken)
+
+
+local function read_wx_file()
+    ngx.req.read_body()
+    local req_body = cjson.decode(ngx.req.get_body_data())
+    ngx.log(ngx.INFO, 'REQ_BODY -> ', ngx.req.get_body_data())
+
+    local files = {}
+    for v in pairs(req_body["payload"]["media_ids"]) do
+        table.insert(files, get_wx_media(v));
+    end
+
+    create_task(files, authHeader, sudoerToken)
+
+end
+
+
+read_wx_file()
+