From d674506717a7489927647d710d54d798744614f5 Mon Sep 17 00:00:00 2001 From: RageCage64 Date: Wed, 10 May 2023 21:42:23 -0400 Subject: [PATCH] Initial commit --- .gitignore | 1 + Cargo.lock | 213 +++++++++++++++++++++++++++++++++++++ Cargo.toml | 9 ++ README.md | 70 ++++++++++++ examples/example.lua | 5 + examples/example_test.yaml | 27 +++++ src/config/config.rs | 83 +++++++++++++++ src/config/mod.rs | 1 + src/main.rs | 75 +++++++++++++ 9 files changed, 484 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 examples/example.lua create mode 100644 examples/example_test.yaml create mode 100644 src/config/config.rs create mode 100644 src/config/mod.rs create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..da106d9 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,213 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "erased-serde" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" +dependencies = [ + "serde", +] + +[[package]] +name = "flb_lua_tester" +version = "0.1.0" +dependencies = [ + "mlua", + "serde", + "serde_yaml", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "lua-src" +version = "544.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708ba3c844d5e9d38def4a09dd871c17c370f519b3c4b7261fbabe4a613a814c" +dependencies = [ + "cc", +] + +[[package]] +name = "luajit-src" +version = "210.4.5+resty2cf5186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b7992a40e602786272d84c6f2beca44a588ededcfd57b48ec6f82008a7cb97" +dependencies = [ + "cc", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mlua" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea8ce6788556a67d90567809c7de94dfef2ff1f47ff897aeee935bcfbcdf5735" +dependencies = [ + "bstr", + "cc", + "erased-serde", + "lua-src", + "luajit-src", + "num-traits", + "once_cell", + "pkg-config", + "rustc-hash", + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "proc-macro2" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "serde" +version = "1.0.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..1725233 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "flb_lua_tester" +version = "0.1.0" +edition = "2021" + +[dependencies] +mlua = { version = "0.8.8", features = ["luajit", "vendored", "serialize" ] } +serde = { version = "1.0.162", features = ["derive"] } +serde_yaml = "0.9.21" diff --git a/README.md b/README.md new file mode 100644 index 0000000..1db060d --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# Fluent Bit Lua Tester + +A tool to test Lua scripts written for Fluent Bit filters. It calls the function and expects return values using the same interface as Fluent Bit does, using LuaJIT like Fluent Bit does as well. + +## Getting Started + +This tool is a very early experiment, so it doesn't have any release infrastructure. The only way to run it currently is to have Rust installed and build/run it with `cargo`. + +The tool accepts the path to the yaml test config as the first argument to the program. + +## Example + +Let's test an extremely simple Lua script `example.lua` that we are using as a Fluent Bit Filter: + +```lua +function filter_entry(tag, timestamp, record) + record["y"] = "z" + return 0, timestamp, record +end +``` + +The test config yaml file specifies a list of Lua scripts to test, in each: +* The file path (relative to the working directory of the binary) +* The function in the script to call +* An array of tests each with + - Test case name + - The input arguments (tag, timestamp, and record) + - The expected output result (code, timestamp, and record) + +Let's write a small set of unit tests for this script in `example_test.yaml`: + +```yaml +scripts: + - file: "example.lua" + call: "filter_entry" + tests: + - name: "adds y key" + input: + tag: "hi" + timestamp: "2014-10-02T15:01:23Z" + record: + w: x + expected: + code: 0 + timestamp: "2014-10-02T15:01:23Z" + record: + w: x + y: z + - name: "resets existing y key" + input: + tag: "hi" + timestamp: "2014-10-02T15:01:23Z" + record: + y: something else + expected: + code: 0 + timestamp: "2014-10-02T15:01:23Z" + record: + y: z +``` + +Run this test with the command `cargo run -- example_test.yaml` + +``` +Running test: "adds y key" +Test Passed + +Running test: "resets existing y key" +Test Passed +``` \ No newline at end of file diff --git a/examples/example.lua b/examples/example.lua new file mode 100644 index 0000000..c707b39 --- /dev/null +++ b/examples/example.lua @@ -0,0 +1,5 @@ + +function filter_entry(tag, timestamp, record) + record["y"] = "z" + return 0, timestamp, record +end \ No newline at end of file diff --git a/examples/example_test.yaml b/examples/example_test.yaml new file mode 100644 index 0000000..6fdb41d --- /dev/null +++ b/examples/example_test.yaml @@ -0,0 +1,27 @@ +scripts: + - file: "examples/example.lua" + call: "filter_entry" + tests: + - name: "adds y key" + input: + tag: "hi" + timestamp: "2014-10-02T15:01:23Z" + record: + w: x + expected: + code: 0 + timestamp: "2014-10-02T15:01:23Z" + record: + w: x + y: z + - name: "resets existing y key" + input: + tag: "hi" + timestamp: "2014-10-02T15:01:23Z" + record: + y: something else + expected: + code: 0 + timestamp: "2014-10-02T15:01:23Z" + record: + y: z \ No newline at end of file diff --git a/src/config/config.rs b/src/config/config.rs new file mode 100644 index 0000000..6a701a6 --- /dev/null +++ b/src/config/config.rs @@ -0,0 +1,83 @@ +use std::collections::HashMap; +use std::fs::File; + +use mlua::{ToLuaMulti, LuaSerdeExt}; +use mlua::prelude::*; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Debug)] +pub struct Config { + pub scripts: Vec, +} + +pub fn load_config(path: String) -> Config { + let f = File::open(path).expect("Could not open file."); + return serde_yaml::from_reader(f).expect("Could not read values."); +} + +#[derive(Deserialize, Debug)] +pub struct ScriptTest { + pub file: String, + pub call: String, + pub tests: Vec, +} + +#[derive(Deserialize, Debug)] +pub struct TestCase { + pub name: String, + pub input: LuaFnInput, + pub expected: LuaFnOutput, +} + +#[derive(Deserialize, Debug)] +pub struct LuaFnInput { + pub tag: String, + pub timestamp: String, + pub record: HashMap, +} + +impl<'lua> ToLuaMulti<'lua> for LuaFnInput { + fn to_lua_multi(self, lua: &'lua mlua::Lua) -> mlua::Result> { + let mut mv = mlua::MultiValue::new(); + + let record = lua.create_table().unwrap(); + for row in self.record.iter() { + let key = lua.to_value(row.0).unwrap(); + let val = lua.to_value(row.1).unwrap(); + record.set(key, val).unwrap(); + } + mv.push_front(mlua::Value::Table(record)); + + mv.push_front(self.timestamp.to_lua(lua).unwrap()); + + mv.push_front(self.tag.to_lua(lua).unwrap()); + + return Ok(mv); + } +} + +#[derive(Deserialize, Debug)] +pub struct LuaFnOutput { + pub code: i64, + pub timestamp: String, + pub record: HashMap, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +pub enum FlbRecordValidType { + String(String), + Number(f64), + Table(HashMap), +} + +impl PartialEq for FlbRecordValidType { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::String(l0), Self::String(r0)) => l0 == r0, + (Self::Number(l0), Self::Number(r0)) => l0 == r0, + (Self::Table(l0), Self::Table(r0)) => l0 == r0, + _ => false, + } + } +} diff --git a/src/config/mod.rs b/src/config/mod.rs new file mode 100644 index 0000000..ef68c36 --- /dev/null +++ b/src/config/mod.rs @@ -0,0 +1 @@ +pub mod config; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..a2c2ee2 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,75 @@ +mod config; + +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::{self, BufRead}; + +use config::config::FlbRecordValidType; +use mlua::prelude::*; + +fn main() { + let mut args = env::args(); + if args.len() == 1 { + println!("Please provide a config file path."); + return; + } + let path = args.nth(1).unwrap(); + let config = config::config::load_config(path); + + for script in config.scripts { + let lua = load_script(script.file); + let f_res = lua.globals().get::<_, mlua::Function>(script.call); + if f_res.is_err() { + continue; + } + let f = f_res.unwrap(); + for tc in script.tests { + println!("Running test: {:?}", tc.name); + let mut test_passed = true; + + let r = f.call::<_, LuaMultiValue>(tc.input); + let mut mv = r.unwrap(); + + let code = i64::from_lua(mv.pop_front().unwrap(), &lua).unwrap(); + if code != tc.expected.code { + test_passed = false; + println!(" expected code: {:?}", tc.expected.code); + println!(" got code: {:?}\n", code); + } + + let timestamp = String::from_lua(mv.pop_front().unwrap(), &lua).unwrap(); + if timestamp != tc.expected.timestamp { + test_passed = false; + println!(" expected timestamp: {:?}", tc.expected.timestamp); + println!(" got timestamp: {:?}\n", timestamp); + } + + let record_value: LuaValue = mv.pop_front().unwrap(); + let record: HashMap = lua.from_value(record_value).unwrap(); + if record != tc.expected.record { + test_passed = false; + println!(" expected record: {:?}", tc.expected.record); + println!(" got record: {:?}\n", record); + } + + if test_passed { + println!("Test Passed\n"); + } else { + println!("Test Failed\n"); + } + } + } +} + +fn load_script(script_path: String) -> Lua { + let lua = Lua::new(); + let mut script_content = "".to_string(); + let file = File::open(script_path).unwrap(); + for line in io::BufReader::new(file).lines() { + script_content += &line.unwrap(); + script_content += "\n"; + } + lua.load(&script_content).exec().unwrap(); + return lua; +}