From 3cc748208498b1f78458bf74e9517181254a9d77 Mon Sep 17 00:00:00 2001 From: Thomas Lynch Date: Sat, 18 Feb 2023 15:13:40 +1100 Subject: [PATCH] Fully convert to data plane api Change global ACL to a map to realtime update with data plane api Change how on setartup servers are registered in lua --- docker-compose.yml | 3 +- haproxy/dataplaneapi.hcl | 11 ++++ haproxy/haproxy.cfg | 50 +++++++++--------- haproxy/map/backends.map | 3 +- haproxy/map/ddos.map | 2 - haproxy/map/ddos_global.map | 0 haproxy/map/hosts.map | 1 + haproxy/template/trace.txt | 1 + nginx/favicon.ico | Bin 0 -> 16958 bytes src/lua/scripts/bot-check.lua | 29 ---------- .../{register.lua => register-bot-check.lua} | 1 - src/lua/scripts/register-servers.lua | 41 ++++++++++++++ src/lua/scripts/templates.lua | 2 +- 13 files changed, 82 insertions(+), 62 deletions(-) create mode 100644 haproxy/map/ddos_global.map create mode 100644 nginx/favicon.ico rename src/lua/scripts/{register.lua => register-bot-check.lua} (92%) create mode 100644 src/lua/scripts/register-servers.lua diff --git a/docker-compose.yml b/docker-compose.yml index 2b98432..36f426f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,7 @@ services: network_mode: host ports: - 80:80 - # - 2000:2000 #runtime api - # - 2001:2001 #dataplaneapi + - 2001:2001 #dataplaneapi build: context: ./ dockerfile: haproxy/Dockerfile diff --git a/haproxy/dataplaneapi.hcl b/haproxy/dataplaneapi.hcl index 5ab0f82..7d61788 100644 --- a/haproxy/dataplaneapi.hcl +++ b/haproxy/dataplaneapi.hcl @@ -1,23 +1,34 @@ config_version = 2 + name = "basedflare" + mode = "single" dataplaneapi { host = "127.0.0.1" port = 2001 + user "admin" { insecure = true password = "admin" } + transaction { transaction_dir = "/tmp/haproxy" } + + resources { + maps_dir = "/etc/haproxy/map" + ssl_certs_dir = "/etc/haproxy/ssl" + } + advertised {} } haproxy { config_file = "/etc/haproxy/haproxy.cfg" haproxy_bin = "/usr/local/sbin/haproxy" + reload { reload_delay = 5 reload_cmd = "service haproxy reload" diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg index daeb518..205ce17 100644 --- a/haproxy/haproxy.cfg +++ b/haproxy/haproxy.cfg @@ -2,7 +2,8 @@ global daemon maxconn 256 log stdout format raw local0 debug - lua-load-per-thread /etc/haproxy/scripts/register.lua + lua-load /etc/haproxy/scripts/register-servers.lua + lua-load-per-thread /etc/haproxy/scripts/register-bot-check.lua stats socket /var/run/haproxy.sock mode 666 level admin stats socket 127.0.0.1:1999 level admin httpclient.ssl.verify none @@ -18,21 +19,21 @@ defaults timeout server 50000ms timeout tarpit 5000ms -# program api - # command dataplaneapi -f /etc/haproxy/dataplaneapi.hcl --update-map-files - # no option start-on-reload -# -# frontend stats-frontend - # bind *:2000 - # option tcplog - # mode tcp - # acl white_list src 127.0.0.1 - # tcp-request connection reject unless white_list - # default_backend stats-backend -# -# backend stats-backend - # mode tcp - # server stats-localhost 127.0.0.1:1999 +program api + command dataplaneapi -f /etc/haproxy/dataplaneapi.hcl --update-map-files + no option start-on-reload + +frontend stats-frontend + bind 127.0.0.1:2000 + option tcplog + mode tcp + acl white_list src 127.0.0.1 + tcp-request connection reject unless white_list + default_backend stats-backend + +backend stats-backend + mode tcp + server stats-localhost 127.0.0.1:1999 frontend http-in @@ -64,7 +65,7 @@ frontend http-in acl is_excluded path /favicon.ico #add more # acl ORs for when ddos_mode_enabled - acl ddos_mode_enabled_override hdr_cnt(xr3la1rfFc) eq 0 + acl ddos_mode_enabled_override str("true"),map(/etc/haproxy/map/ddos_global.map) -m found acl ddos_mode_enabled hdr(host),lower,map(/etc/haproxy/map/ddos.map) -m bool acl ddos_mode_enabled base,map(/etc/haproxy/map/ddos.map) -m bool @@ -97,6 +98,12 @@ frontend http-in http-response set-header X-Cache-Status HIT if !{ srv_id -m found } http-response set-header X-Cache-Status MISS if { srv_id -m found } + # simple example cache for files + http-request set-var(txn.path) path + acl can_cache var(txn.path) -i -m end .png .jpg .jpeg .jpe .ico .webmanifest .xml .apng .bmp .webp .pjpeg .jfif .gif .mp4 .webm .mov .mkv .svg .m4a .aac .flac .mp3 .ogg .wav .opus .txt .pdf .sid + http-request cache-use basic_cache if can_cache + http-response cache-store basic_cache if can_cache + default_backend servers cache basic_cache @@ -105,15 +112,6 @@ cache basic_cache max-age 86400 backend servers - - # simple example cache for files - http-request set-var(txn.path) path - acl can_cache var(txn.path) -i -m end .png .jpg .jpeg .jpe .ico .webmanifest .xml .apng .bmp .webp .pjpeg .jfif .gif .mp4 .webm .mov .mkv .svg .m4a .aac .flac .mp3 .ogg .wav .opus .txt .pdf .sid - http-request cache-use basic_cache if can_cache - http-response cache-store basic_cache if can_cache - - # placeholder servers, activated by LUA or the control panel - server-template websrv 1-100 0.0.0.0:80 check disabled # use server based on hostname use-server %[req.hdr(host),lower,map(/etc/haproxy/map/backends.map)] if TRUE diff --git a/haproxy/map/backends.map b/haproxy/map/backends.map index 4479d20..ac09232 100644 --- a/haproxy/map/backends.map +++ b/haproxy/map/backends.map @@ -1 +1,2 @@ -localhost websrv1 +127.0.0.1 websrv1 +localhost websrv2 diff --git a/haproxy/map/ddos.map b/haproxy/map/ddos.map index 4baa168..e69de29 100644 --- a/haproxy/map/ddos.map +++ b/haproxy/map/ddos.map @@ -1,2 +0,0 @@ -localhost 1 -localhost/test 2 diff --git a/haproxy/map/ddos_global.map b/haproxy/map/ddos_global.map new file mode 100644 index 0000000..e69de29 diff --git a/haproxy/map/hosts.map b/haproxy/map/hosts.map index ce38125..fc61161 100644 --- a/haproxy/map/hosts.map +++ b/haproxy/map/hosts.map @@ -1 +1,2 @@ +127.0.0.1 127.0.0.1:81 localhost 127.0.0.1:81 diff --git a/haproxy/template/trace.txt b/haproxy/template/trace.txt index 48410ef..91fc547 100644 --- a/haproxy/template/trace.txt +++ b/haproxy/template/trace.txt @@ -5,6 +5,7 @@ uag=%[req.fhdr(user-agent)] http=%HV tls=%[ssl_fc] tlsv=%sslv +tlsf=%[ssl_c_sha1,hex] sni=%[ssl_fc_sni] vey_id=%[env(RAY_ID)] expiry=%[env(CHALLENGE_EXPIRY)] diff --git a/nginx/favicon.ico b/nginx/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..00f3a3ae65fb8c4bcad40d8942ac37bc722bd9e3 GIT binary patch literal 16958 zcmeI3T}&0%7RS%^BQeJK1!5XXY7<+uZETySDcGREgQN)t;h~lXORZYJ(kNnLT4G~O z+CEfD+M+(RHpECYY0^}zm0B&_i=r10xgt`F$faHpg#uSlLBMtAx4Cl|=FH4Fhl9#N zH(6(9_SdZc+H3E<)?S0l^*8M=HrA#6&T`F(ce%n`F4r85v@F*)jmTFiw*O-MmB&f$ z-Mc4^jg4~Q!i7hIu3ft}Xw;jUnj|(hR;Ev%?nogaArcxIDqOwQ^`)GChO;kUc$+qD zn#9M)%k}Hm-7;X@;a8g%GIi=ynK5IAB~6+%N!1xYe!OM5T|29M_~3mI_f~cM$%7~Q zf=9b{V51IL!K^3l8HWiICP-dho+_)VsuCknCo?lswVywKzMMUKc2K#ouDAQ@>S{S~ z;DAh-GDQ*he(LG-Me?IcO)bv zNMB!{N4Y;46%`fkImCS+30~j{{P1X(zy?Mq{HIT!wregsckWd0ShQ%7+`W6(ww^Bq z<>lpW{M-kU;02z*507>UY$FAKSy`D||G5t&1tWguTwGk7+IN`yfuvx>-_z40t*xz! zxDO-+BmO|}n%h2X{F?bFg@rfd+ix$+ci+|fMY*~EmLGojkDNT&B6W40a_3I38J^eY zhmF6l@1DH(-Wkn~IjGql`F_(APaKk`o;oZqz4WUrU3yBs`l?ob|NW-6$7+K0Aq6{r zWEL8lukFhRwLS5mx_|M-qw>*5=jHO{POlOe>G;8BM|S#xET9Doilm_6n&zX~i{U}x z-?eL(+x{`G*-rW&9{!Weo%@SC_uK&S{LC{)WY(<1GIQo3HRk%bo8>|Nv(FaDzI_dY zWPs1ACwS5AA9%D&-t3>TW5-HLN{Xyow@%isT|0=@uU{`QF)<2%L_~zFTD3~{@87Rp zWrrW$Q&LJaTfALrYHkjqii$QVE^bz9-zT5c$Qy4I%ac$3q{iGV1M~nH962)Jx0aNY zxcOYYdbPk4_8&aLtC@c=fenmS_AkC8?fs#ZD_1(=XFYu9oigd|9T+RSv1x0&E8l;A zMd>?u%`!li@$r9`D_3r-F6!#)WcKXYe(|eg1FMz)Gj7~CNli_ajEoG~v}u!~HEY($ z^5x4tC@n2b_U_&56@OYkeONKh|{hxnDDo@cB}QCId7Z;JZEJ5YmhBqhu54R zO!yi9=;+@p$A8O~b7p8hJqHv1vNHSm|Ml0kKH)W23MTv?e_ZV`_N@Q-iA6;>%+P## z4i@~!j$Kzi19;7IAKR$CeQ0N|4-0=!k9Yf*0p#q1vyfL`^<4KuLh?0#>1R1~sL`jf zwyNX{ey8()t3JuI+y7asQ&Ue%Q&X4x^;fqCwX}50pMT!cVkvFPzgxDfROz?b{^uFr z=!-A@5MO0!#j7{_kJw0VZmtlI;7r;ZH8eD+*Vy4lf1iH(XDv4Ko4o#dp+;PFDpdK{ zx%k3iVfpT}4_)VJj|}I{J1XCNb6L8(`~8T{&Q2A7^2VRB986%di~s2U-{i@YwRhUb zLGcqH>fuw6?C^sF42+)>nddom$$S3!UnD)fQYtIk)O+}8Z*P|k8#Z|GsPpB-J3Qk9 zTKRvz;Kx3uJw6nDZQi_D@#TbHA4`9(F)@Xbo&ATLKmXrBzH@VPv-0P{!opO{RL4sE z6rSJ>9`$_l)d@fH)AgTtA~7i=k+)^b7WIy(s3XdlXe|`MXVSGU&kq4Q_o;JzJ zC35CWtCFqp2mJ7aUK@F1U4tj^!=oMlPWlgiWaqU0*~LHY#vg1iyl_D zz4_)TW#2|d7N|WAo!51Qd)l$yaW3L5J?!|W9e&1}HT~ADUUde-`lSy5z05PFvc&!~*Qj5l0Jt2ADV)_JDCNa6fl$fwGgVlwdUBzkRz`SR2uM zeGYJCJ>X2tDvc)m;M=yX+5FDtMR%AHwW7m?*aRu^na&nZ7 zNvyG}t4q}(-`oE0>;6Yy*gCJh=IQ_U+fx4=H2<;t;1gc-_$NG)toR2THZHcVRs4(C zU}9pTtXQ$aBc?=Nbab@BA08gA*>%f?J^sbo_Th(BgVue04`4oE50iR(d(~K^{Xh1_ z6`rvF;n9wNu#MFCXJg|XweEAyVP5-qo-?OFJD2G9gRfq_Dv^>wEtPA5^*Rzxw)m z*}i?dn@9NcMqmQlP~x9T=^TFS=0Cja-#_rpgPE}NC`Y%=>%Fe`tb2)xC)7Ea0}DPC zc{~4NZNQiIH#y-42V-hSI!^Odmj~ZZ@GaDEBwx-y?+1P}Hmhgk17^mXd(Qkm`>aOr ziH0kUMEo;n=8OLQ+siNis=k-uJb?89y&|!{*Vfi*ezfP_?~iQm7k+FOVlqSd{xL7F zL4N%4s+>5{BK7s1>a4_wiT_R6xN)OeyV&OgN%sqXAUt_%tABI0WXTfE_UZOk-Ld>Z z;qU0^kd~GfEuLe4X5+}Hp)e~eOYQH;$;m;1pSen0VBWlWT1;o30m;yAgAG4Anv|5J z#%#xq9YX_cXcdh3(N%0xcn>r^X#B($0!jELd}j&%?Ck77x>9ZFgTCmKz5_{MbMpU) z|L{#A^;j#1LdC_!vT)%-#WQibK>DIBeSn`n>3b+V(T2XjW*7e?R>pfMM@E6rm@#8i zU;IyEApOu*|G!51474w>fzj;$^S_{+A0PwDhJ)}ihoc|*7|y=vZ}H;AYJWD)C1k*v V$fH2mCXZ9U1ho0^I6cA=_#aZjp(_9Y literal 0 HcmV?d00001 diff --git a/src/lua/scripts/bot-check.lua b/src/lua/scripts/bot-check.lua index bca23bd..38f26a7 100644 --- a/src/lua/scripts/bot-check.lua +++ b/src/lua/scripts/bot-check.lua @@ -56,35 +56,6 @@ else captcha_backend_name = "recaptcha" end --- setup initial server backends based on hosts.map into backends.map -function _M.setup_servers() - if pow_difficulty < 8 then - error("POW_DIFFICULTY must be > 8. Around 16-32 is better") - end - local backend_name = os.getenv("BACKEND_NAME") - local server_prefix = os.getenv("SERVER_PREFIX") - if backend_name == nil or server_prefix == nil then - return; - end - local hosts_map = Map.new("/etc/haproxy/map/hosts.map", Map._str); - local handle = io.open("/etc/haproxy/map/hosts.map", "r") - local line = handle:read("*line") - local counter = 1 - while line do - local domain, backend_host = line:match("([^%s]+)%s+([^%s]+)") - local port_index = backend_host:match'^.*():' - local backend_hostname = backend_host:sub(0, port_index-1) - local backend_port = backend_host:sub(port_index + 1) - core.set_map("/etc/haproxy/map/backends.map", domain, server_prefix..counter) - local proxy = core.proxies[backend_name].servers[server_prefix..counter] - proxy:set_addr(backend_hostname, backend_port) - proxy:set_ready() - line = handle:read("*line") - counter = counter + 1 - end - handle:close() -end - -- kill a tor circuit function _M.kill_tor_circuit(txn) local ip = txn.sf:src() diff --git a/src/lua/scripts/register.lua b/src/lua/scripts/register-bot-check.lua similarity index 92% rename from src/lua/scripts/register.lua rename to src/lua/scripts/register-bot-check.lua index 6ba84ef..8d6a58c 100644 --- a/src/lua/scripts/register.lua +++ b/src/lua/scripts/register-bot-check.lua @@ -7,4 +7,3 @@ core.register_action("captcha-check", { 'http-req', }, bot_check.check_captcha_s core.register_action("pow-check", { 'http-req', }, bot_check.check_pow_status) core.register_action("decide-checks-necessary", { 'http-req', }, bot_check.decide_checks_necessary) core.register_action("kill-tor-circuit", { 'http-req', }, bot_check.kill_tor_circuit) -core.register_init(bot_check.setup_servers) diff --git a/src/lua/scripts/register-servers.lua b/src/lua/scripts/register-servers.lua new file mode 100644 index 0000000..92ec7f9 --- /dev/null +++ b/src/lua/scripts/register-servers.lua @@ -0,0 +1,41 @@ +package.path = package.path .. "./?.lua;/etc/haproxy/scripts/?.lua;/etc/haproxy/libs/?.lua" + +local pow_difficulty = tonumber(os.getenv("POW_DIFFICULTY") or 18) + +-- setup initial server backends based on hosts.map +function setup_servers() + if pow_difficulty < 8 then + error("POW_DIFFICULTY must be > 8. Around 16-32 is better") + end + local backend_name = os.getenv("BACKEND_NAME") + local server_prefix = os.getenv("SERVER_PREFIX") + if backend_name == nil or server_prefix == nil then + return; + end + local handle = io.open("/etc/haproxy/map/hosts.map", "r") + local line = handle:read("*line") + local counter = 1 + -- NOTE: using tcp socket to interact with runtime API because lua can't add servers + local tcp = core.tcp(); + tcp:settimeout(1); + tcp:connect("127.0.0.1", 2000); --TODO: configurable port + while line do + local domain, backend_host = line:match("([^%s]+)%s+([^%s]+)") + -- local host_split = utils.split(backend_host, ":") + -- local backend_hostname = host_split[1] + -- local backend_port = host_split[2] + core.set_map("/etc/haproxy/map/backends.map", domain, server_prefix..counter) + -- local proxy = core.proxies[backend_name].servers[server_prefix..counter] + -- proxy:set_addr(backend_hostname, backend_port) + -- proxy:set_ready() + local server_name = "servers/websrv"..counter + tcp:send(string.format("add server %s %s check\n", server_name, backend_host)) + tcp:send(string.format("enable server %s\n", server_name)) + line = handle:read("*line") + counter = counter + 1 + end + handle:close() + tcp:close() +end + +core.register_task(setup_servers) diff --git a/src/lua/scripts/templates.lua b/src/lua/scripts/templates.lua index 03c9b4c..2280f51 100644 --- a/src/lua/scripts/templates.lua +++ b/src/lua/scripts/templates.lua @@ -13,7 +13,7 @@ _M.body = [[ .h-captcha,.g-recaptcha{min-height:85px;display:block} .red{color:red;font-weight:bold} .left{text-align:left} - .powstatus{color:green;font-weight:bold} + .powstatus{color:green;font-size:small;} a,a:visited{color:var(--text-color)} body,html{height:100%%;text-align:center;} body{display:flex;flex-direction:column;background-color:var(--bg-color);color:var(--text-color);font-family:Helvetica,Arial,sans-serif;max-width:60em;margin:0 auto;padding: 0 20px}