From 2405a79ace38e4a4a7e6c36f77b6b0c9cdd511f2 Mon Sep 17 00:00:00 2001 From: Lykin <137850705+tiny-craft@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:55:46 +0800 Subject: [PATCH] feat: add PHP Serialized/Pickle as build-in decoder. #64 #87 --- backend/services/preferences_service.go | 15 +- backend/types/view_type.go | 2 + backend/utils/convert/base64_convert.go | 4 + backend/utils/convert/binary_convert.go | 4 + backend/utils/convert/brotli_convert.go | 4 + backend/utils/convert/cmd_convert.go | 17 ++ backend/utils/convert/convert.go | 191 +++++------------- backend/utils/convert/deflate_convert.go | 4 + backend/utils/convert/gzip_convert.go | 4 + backend/utils/convert/hex_convert.go | 4 + backend/utils/convert/json_convert.go | 4 + backend/utils/convert/msgpack_convert.go | 8 +- backend/utils/convert/php_convert.go | 89 ++++++++ backend/utils/convert/pickle_convert.go | 86 ++++++++ backend/utils/convert/xml_convert.go | 4 + backend/utils/convert/yaml_convert.go | 4 + backend/utils/convert/zstd_convert.go | 4 + frontend/src/App.vue | 1 + .../content_value/FormatSelector.vue | 27 ++- frontend/src/consts/value_view_type.js | 3 +- frontend/src/stores/preferences.js | 16 ++ 21 files changed, 342 insertions(+), 153 deletions(-) create mode 100644 backend/utils/convert/php_convert.go create mode 100644 backend/utils/convert/pickle_convert.go diff --git a/backend/services/preferences_service.go b/backend/services/preferences_service.go index 0f1ff69..43c03df 100644 --- a/backend/services/preferences_service.go +++ b/backend/services/preferences_service.go @@ -20,7 +20,6 @@ import ( type preferencesService struct { pref *storage2.PreferencesStorage clientVersion string - customDecoder []convutil.CmdConvert } var preferences *preferencesService @@ -101,6 +100,20 @@ func (p *preferencesService) GetFontList() (resp types.JSResp) { return } +func (p *preferencesService) GetBuildInDecoder() (resp types.JSResp) { + buildinDecoder := make([]string, 0, len(convutil.BuildInDecoders)) + for name, convert := range convutil.BuildInDecoders { + if convert.Enable() { + buildinDecoder = append(buildinDecoder, name) + } + } + resp.Data = map[string]any{ + "decoder": buildinDecoder, + } + resp.Success = true + return +} + func (p *preferencesService) SetAppVersion(ver string) { if !strings.HasPrefix(ver, "v") { p.clientVersion = "v" + ver diff --git a/backend/types/view_type.go b/backend/types/view_type.go index ddc5206..161aead 100644 --- a/backend/types/view_type.go +++ b/backend/types/view_type.go @@ -14,3 +14,5 @@ const DECODE_DEFLATE = "Deflate" const DECODE_ZSTD = "ZStd" const DECODE_BROTLI = "Brotli" const DECODE_MSGPACK = "Msgpack" +const DECODE_PHP = "PHP" +const DECODE_PICKLE = "Pickle" diff --git a/backend/utils/convert/base64_convert.go b/backend/utils/convert/base64_convert.go index 70a19e2..d2e116f 100644 --- a/backend/utils/convert/base64_convert.go +++ b/backend/utils/convert/base64_convert.go @@ -7,6 +7,10 @@ import ( type Base64Convert struct{} +func (Base64Convert) Enable() bool { + return true +} + func (Base64Convert) Encode(str string) (string, bool) { return base64.StdEncoding.EncodeToString([]byte(str)), true } diff --git a/backend/utils/convert/binary_convert.go b/backend/utils/convert/binary_convert.go index 376fe85..b2fc6b7 100644 --- a/backend/utils/convert/binary_convert.go +++ b/backend/utils/convert/binary_convert.go @@ -8,6 +8,10 @@ import ( type BinaryConvert struct{} +func (BinaryConvert) Enable() bool { + return true +} + func (BinaryConvert) Encode(str string) (string, bool) { var result strings.Builder total := len(str) diff --git a/backend/utils/convert/brotli_convert.go b/backend/utils/convert/brotli_convert.go index 399f8bc..69210ec 100644 --- a/backend/utils/convert/brotli_convert.go +++ b/backend/utils/convert/brotli_convert.go @@ -9,6 +9,10 @@ import ( type BrotliConvert struct{} +func (BrotliConvert) Enable() bool { + return true +} + func (BrotliConvert) Encode(str string) (string, bool) { var compress = func(b []byte) (string, error) { var buf bytes.Buffer diff --git a/backend/utils/convert/cmd_convert.go b/backend/utils/convert/cmd_convert.go index a37015f..0986560 100644 --- a/backend/utils/convert/cmd_convert.go +++ b/backend/utils/convert/cmd_convert.go @@ -2,7 +2,10 @@ package convutil import ( "encoding/base64" + "github.com/vrischmann/userdir" + "os" "os/exec" + "path" "strings" sliceutil "tinyrdm/backend/utils/slice" ) @@ -18,6 +21,10 @@ type CmdConvert struct { const replaceholder = "{VALUE}" +func (c CmdConvert) Enable() bool { + return true +} + func (c CmdConvert) Encode(str string) (string, bool) { base64Content := base64.StdEncoding.EncodeToString([]byte(str)) var containHolder bool @@ -73,3 +80,13 @@ func (c CmdConvert) Decode(str string) (string, bool) { } return string(outputContent[:n]), true } + +func (c CmdConvert) writeExecuteFile(content []byte, filename string) (string, error) { + filepath := path.Join(userdir.GetConfigHome(), "TinyRDM", "decoder", filename) + _ = os.Mkdir(path.Dir(filepath), 0777) + err := os.WriteFile(filepath, content, 0777) + if err != nil { + return "", err + } + return filepath, nil +} diff --git a/backend/utils/convert/convert.go b/backend/utils/convert/convert.go index aa47101..c713d3a 100644 --- a/backend/utils/convert/convert.go +++ b/backend/utils/convert/convert.go @@ -8,6 +8,7 @@ import ( ) type DataConvert interface { + Enable() bool Encode(string) (string, bool) Decode(string) (string, bool) } @@ -24,8 +25,29 @@ var ( zstdConv ZStdConvert brotliConv BrotliConvert msgpackConv MsgpackConvert + phpConv = NewPhpConvert() + pickleConv = NewPickleConvert() ) +var BuildInFormatters = map[string]DataConvert{ + types.FORMAT_JSON: jsonConv, + types.FORMAT_YAML: yamlConv, + types.FORMAT_XML: xmlConv, + types.FORMAT_HEX: hexConv, + types.FORMAT_BINARY: binaryConv, +} + +var BuildInDecoders = map[string]DataConvert{ + types.DECODE_BASE64: base64Conv, + types.DECODE_GZIP: gzipConv, + types.DECODE_DEFLATE: deflateConv, + types.DECODE_ZSTD: zstdConv, + types.DECODE_BROTLI: brotliConv, + types.DECODE_MSGPACK: msgpackConv, + types.DECODE_PHP: phpConv, + types.DECODE_PICKLE: pickleConv, +} + // ConvertTo convert string to specified type // @param decodeType empty string indicates automatic detection // @param formatType empty string indicates automatic detection @@ -59,59 +81,17 @@ func ConvertTo(str, decodeType, formatType string, customDecoder []CmdConvert) ( func decodeWith(str, decodeType string, customDecoder []CmdConvert) (value, resultDecode string) { if len(decodeType) > 0 { - switch decodeType { - case types.DECODE_NONE: - value = str + value = str - case types.DECODE_BASE64: - if base64Str, ok := base64Conv.Decode(str); ok { - value = base64Str - } else { - value = str + if buildinDecoder, ok := BuildInDecoders[decodeType]; ok { + if decodedStr, ok := buildinDecoder.Decode(str); ok { + value = decodedStr } - - case types.DECODE_GZIP: - if gzipStr, ok := gzipConv.Decode(str); ok { - value = gzipStr - } else { - value = str - } - - case types.DECODE_DEFLATE: - if falteStr, ok := deflateConv.Decode(str); ok { - value = falteStr - } else { - value = str - } - - case types.DECODE_ZSTD: - if zstdStr, ok := zstdConv.Decode(str); ok { - value = zstdStr - } else { - value = str - } - - case types.DECODE_BROTLI: - if brotliStr, ok := brotliConv.Decode(str); ok { - value = brotliStr - } else { - value = str - } - - case types.DECODE_MSGPACK: - if msgpackStr, ok := msgpackConv.Decode(str); ok { - value = msgpackStr - } else { - value = str - } - - default: + } else if decodeType != types.DECODE_NONE { for _, decoder := range customDecoder { if decoder.Name == decodeType { if decodedStr, ok := decoder.Decode(str); ok { value = decodedStr - } else { - value = str } break } @@ -167,6 +147,16 @@ func autoDecode(str string, customDecoder []CmdConvert) (value, resultDecode str return } + if value, ok = phpConv.Decode(str); ok { + resultDecode = types.DECODE_PHP + return + } + + if value, ok = pickleConv.Decode(str); ok { + resultDecode = types.DECODE_PICKLE + return + } + // try decode with custom decoder for _, decoder := range customDecoder { if decoder.Auto { @@ -186,31 +176,10 @@ func autoDecode(str string, customDecoder []CmdConvert) (value, resultDecode str func viewAs(str, formatType string) (value, resultFormat string) { if len(formatType) > 0 { - switch formatType { - default: - fallthrough - case types.FORMAT_RAW, types.FORMAT_YAML, types.FORMAT_XML: - value = str - - case types.FORMAT_JSON: - if jsonStr, ok := jsonConv.Decode(str); ok { - value = jsonStr - } else { - value = str - } - - case types.FORMAT_HEX: - if hexStr, ok := hexConv.Decode(str); ok { - value = hexStr - } else { - value = str - } - - case types.FORMAT_BINARY: - if binStr, ok := binaryConv.Decode(str); ok { - value = binStr - } else { - value = str + value = str + if buildinFormatter, ok := BuildInFormatters[formatType]; ok { + if formattedStr, ok := buildinFormatter.Decode(str); ok { + value = formattedStr } } resultFormat = formatType @@ -254,87 +223,29 @@ func autoViewAs(str string) (value, resultFormat string) { func SaveAs(str, format, decode string, customDecoder []CmdConvert) (value string, err error) { value = str - switch format { - case types.FORMAT_JSON: - if jsonStr, ok := jsonConv.Encode(str); ok { - value = jsonStr + if buildingFormatter, ok := BuildInFormatters[format]; ok { + if formattedStr, ok := buildingFormatter.Encode(str); ok { + value = formattedStr } else { - err = errors.New("invalid json data") - return - } - - case types.FORMAT_HEX: - if hexStr, ok := hexConv.Encode(str); ok { - value = hexStr - } else { - err = errors.New("invalid hex data") - return - } - - case types.FORMAT_BINARY: - if binStr, ok := binaryConv.Encode(str); ok { - value = binStr - } else { - err = errors.New("invalid binary data") + err = errors.New("invalid " + format + " data") return } } - switch decode { - case types.DECODE_NONE: - return - - case types.DECODE_BASE64: - value, _ = base64Conv.Encode(value) - return - - case types.DECODE_GZIP: - if gzipStr, ok := gzipConv.Encode(str); ok { - value = gzipStr + if buildinDecoder, ok := BuildInDecoders[decode]; ok { + if encodedValue, ok := buildinDecoder.Encode(str); ok { + value = encodedValue } else { - err = errors.New("fail to build gzip") + err = errors.New("fail to build " + decode) } return - - case types.DECODE_DEFLATE: - if deflateStr, ok := deflateConv.Encode(str); ok { - value = deflateStr - } else { - err = errors.New("fail to build deflate") - } - return - - case types.DECODE_ZSTD: - if zstdStr, ok := zstdConv.Encode(str); ok { - value = zstdStr - } else { - err = errors.New("fail to build zstd") - } - return - - case types.DECODE_BROTLI: - if brotliStr, ok := brotliConv.Encode(str); ok { - value = brotliStr - } else { - err = errors.New("fail to build brotli") - } - return - - case types.DECODE_MSGPACK: - if msgpackStr, ok := msgpackConv.Encode(str); ok { - value = msgpackStr - } else { - err = errors.New("fail to build msgpack") - } - return - - default: + } else if decode != types.DECODE_NONE { for _, decoder := range customDecoder { if decoder.Name == decode { if encodedStr, ok := decoder.Encode(str); ok { value = encodedStr } else { - value = str + err = errors.New("fail to build " + decode) } return } diff --git a/backend/utils/convert/deflate_convert.go b/backend/utils/convert/deflate_convert.go index d1e2678..f9a3e2c 100644 --- a/backend/utils/convert/deflate_convert.go +++ b/backend/utils/convert/deflate_convert.go @@ -9,6 +9,10 @@ import ( type DeflateConvert struct{} +func (d DeflateConvert) Enable() bool { + return true +} + func (d DeflateConvert) Encode(str string) (string, bool) { var compress = func(b []byte) (string, error) { var buf bytes.Buffer diff --git a/backend/utils/convert/gzip_convert.go b/backend/utils/convert/gzip_convert.go index dc3c856..40d73c6 100644 --- a/backend/utils/convert/gzip_convert.go +++ b/backend/utils/convert/gzip_convert.go @@ -9,6 +9,10 @@ import ( type GZipConvert struct{} +func (GZipConvert) Enable() bool { + return true +} + func (GZipConvert) Encode(str string) (string, bool) { var compress = func(b []byte) (string, error) { var buf bytes.Buffer diff --git a/backend/utils/convert/hex_convert.go b/backend/utils/convert/hex_convert.go index c2c67e2..b0b418b 100644 --- a/backend/utils/convert/hex_convert.go +++ b/backend/utils/convert/hex_convert.go @@ -7,6 +7,10 @@ import ( type HexConvert struct{} +func (HexConvert) Enable() bool { + return true +} + func (HexConvert) Encode(str string) (string, bool) { hexStrArr := strings.Split(str, "\\x") hexStr := strings.Join(hexStrArr, "") diff --git a/backend/utils/convert/json_convert.go b/backend/utils/convert/json_convert.go index aca2eec..73ab02c 100644 --- a/backend/utils/convert/json_convert.go +++ b/backend/utils/convert/json_convert.go @@ -8,6 +8,10 @@ import ( type JsonConvert struct{} +func (JsonConvert) Enable() bool { + return true +} + func (JsonConvert) Decode(str string) (string, bool) { trimedStr := strings.TrimSpace(str) if (strings.HasPrefix(trimedStr, "{") && strings.HasSuffix(trimedStr, "}")) || diff --git a/backend/utils/convert/msgpack_convert.go b/backend/utils/convert/msgpack_convert.go index 8452d31..058ef91 100644 --- a/backend/utils/convert/msgpack_convert.go +++ b/backend/utils/convert/msgpack_convert.go @@ -7,6 +7,10 @@ import ( type MsgpackConvert struct{} +func (MsgpackConvert) Enable() bool { + return true +} + func (MsgpackConvert) Encode(str string) (string, bool) { var obj map[string]any if err := json.Unmarshal([]byte(str), &obj); err == nil { @@ -31,7 +35,9 @@ func (MsgpackConvert) Decode(str string) (string, bool) { var obj map[string]any if err := msgpack.Unmarshal([]byte(str), &obj); err == nil { if b, err := json.Marshal(obj); err == nil { - return string(b), true + if len(b) >= 10 { + return string(b), true + } } } diff --git a/backend/utils/convert/php_convert.go b/backend/utils/convert/php_convert.go new file mode 100644 index 0000000..041e638 --- /dev/null +++ b/backend/utils/convert/php_convert.go @@ -0,0 +1,89 @@ +package convutil + +import ( + "os/exec" +) + +type PhpConvert struct { + CmdConvert +} + +const phpDecodeCode = ` += 3: + action = sys.argv[1].lower() + content = sys.argv[2] + + try: + if action == 'decode': + decoded = base64.b64decode(content) + obj = pickle.loads(decoded) + unserialized = json.dumps(obj, ensure_ascii=False) + print(base64.b64encode(unserialized.encode('utf-8')).decode('utf-8')) + elif action == 'encode': + decoded = base64.b64decode(content) + obj = json.loads(decoded) + serialized = pickle.dumps(obj) + print(base64.b64encode(serialized).decode('utf-8')) + except: + print('[RDM-ERROR]') + else: + print('[RDM-ERROR]') + +` + +func NewPickleConvert() *PickleConvert { + c := CmdConvert{ + Name: "Pickle", + Auto: true, + } + c.DecodePath, c.EncodePath = "python3", "python3" + var err error + if err = exec.Command(c.DecodePath, "--version").Err; err != nil { + c.DecodePath, c.EncodePath = "python", "python" + if err = exec.Command(c.DecodePath, "--version").Err; err != nil { + return nil + } + } + // check if pickle available + if err = exec.Command(c.DecodePath, "-c", "import pickle").Err; err != nil { + return nil + } + var filepath string + if filepath, err = c.writeExecuteFile([]byte(pickleDecodeCode), "pickle_decoder.py"); err != nil { + return nil + } + c.DecodeArgs = []string{filepath, "decode"} + c.EncodeArgs = []string{filepath, "encode"} + + return &PickleConvert{ + CmdConvert: c, + } +} + +func (p *PickleConvert) Enable() bool { + if p == nil { + return false + } + return true +} + +func (p *PickleConvert) Encode(str string) (string, bool) { + if !p.Enable() { + return str, false + } + return p.CmdConvert.Encode(str) +} + +func (p *PickleConvert) Decode(str string) (string, bool) { + if !p.Enable() { + return str, false + } + return p.CmdConvert.Decode(str) +} diff --git a/backend/utils/convert/xml_convert.go b/backend/utils/convert/xml_convert.go index c1c4342..fca4497 100644 --- a/backend/utils/convert/xml_convert.go +++ b/backend/utils/convert/xml_convert.go @@ -7,6 +7,10 @@ import ( type XmlConvert struct{} +func (XmlConvert) Enable() bool { + return true +} + func (XmlConvert) Encode(str string) (string, bool) { return str, true } diff --git a/backend/utils/convert/yaml_convert.go b/backend/utils/convert/yaml_convert.go index 0b1d9a6..512c4c2 100644 --- a/backend/utils/convert/yaml_convert.go +++ b/backend/utils/convert/yaml_convert.go @@ -6,6 +6,10 @@ import ( type YamlConvert struct{} +func (YamlConvert) Enable() bool { + return true +} + func (YamlConvert) Encode(str string) (string, bool) { return str, true } diff --git a/backend/utils/convert/zstd_convert.go b/backend/utils/convert/zstd_convert.go index a768a44..bae50c0 100644 --- a/backend/utils/convert/zstd_convert.go +++ b/backend/utils/convert/zstd_convert.go @@ -9,6 +9,10 @@ import ( type ZStdConvert struct{} +func (ZStdConvert) Enable() bool { + return true +} + func (ZStdConvert) Encode(str string) (string, bool) { var compress = func(b []byte) (string, error) { var buf bytes.Buffer diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 01b220a..ce4dada 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -31,6 +31,7 @@ onMounted(async () => { try { initializing.value = true await prefStore.loadFontList() + await prefStore.loadBuildInDecoder() await connectionStore.initConnections() if (prefStore.autoCheckUpdate) { prefStore.checkForUpdate() diff --git a/frontend/src/components/content_value/FormatSelector.vue b/frontend/src/components/content_value/FormatSelector.vue index d6aaddb..e481d92 100644 --- a/frontend/src/components/content_value/FormatSelector.vue +++ b/frontend/src/components/content_value/FormatSelector.vue @@ -3,7 +3,7 @@ import { decodeTypes, formatTypes } from '@/consts/value_view_type.js' import Code from '@/components/icons/Code.vue' import Conversion from '@/components/icons/Conversion.vue' import DropdownSelector from '@/components/common/DropdownSelector.vue' -import { isEmpty, map, some } from 'lodash' +import { includes, isEmpty, map, pull, some, values } from 'lodash' import { computed } from 'vue' import usePreferencesStore from 'stores/preferences.js' @@ -26,17 +26,24 @@ const formatTypeOption = computed(() => { }) const decodeTypeOption = computed(() => { - const customTypes = [] - // has custom decoder - if (!isEmpty(prefStore.decoder)) { - for (const decoder of prefStore.decoder) { - // types[decoder.name] = types[decoder.name] || decoder.name - if (!decodeTypes.hasOwnProperty(decoder.name)) { - customTypes.push(decoder.name) - } + const buildinTypes = [decodeTypes.NONE], + customTypes = [] + const typs = values(decodeTypes) + // build-in decoder + for (const typ of typs) { + if (includes(prefStore.buildInDecoder, typ)) { + buildinTypes.push(typ) } } - return [map(decodeTypes, (t) => t), customTypes] + // custom decoder + if (!isEmpty(prefStore.decoder)) { + for (const decoder of prefStore.decoder) { + // replace build-in decoder if name conflicted + pull(buildinTypes, decoder.name) + customTypes.push(decoder.name) + } + } + return [buildinTypes, customTypes] }) const emit = defineEmits(['formatChanged', 'update:decode', 'update:format']) diff --git a/frontend/src/consts/value_view_type.js b/frontend/src/consts/value_view_type.js index b8f4f31..dc9a9ad 100644 --- a/frontend/src/consts/value_view_type.js +++ b/frontend/src/consts/value_view_type.js @@ -23,6 +23,7 @@ export const decodeTypes = { ZSTD: 'ZStd', BROTLI: 'Brotli', MSGPACK: 'Msgpack', - // PHP: 'PHP', + PHP: 'PHP', + PICKLE: 'Pickle', // Java: 'Java', } diff --git a/frontend/src/stores/preferences.js b/frontend/src/stores/preferences.js index 11d0231..cc724cc 100644 --- a/frontend/src/stores/preferences.js +++ b/frontend/src/stores/preferences.js @@ -3,6 +3,7 @@ import { lang } from '@/langs/index.js' import { cloneDeep, findIndex, get, isEmpty, join, map, merge, pick, set, some, split } from 'lodash' import { CheckForUpdate, + GetBuildInDecoder, GetFontList, GetPreferences, RestorePreferences, @@ -66,6 +67,7 @@ const usePreferencesStore = defineStore('preferences', { fontSize: 14, cursorStyle: 'block', }, + buildInDecoder: [], decoder: [], lastPref: {}, fontList: [], @@ -320,6 +322,20 @@ const usePreferencesStore = defineStore('preferences', { return this.fontList }, + /** + * get all available build-in decoder + * @return {Promise} + */ + async loadBuildInDecoder() { + const { success, data } = await GetBuildInDecoder() + if (success) { + const { decoder = [] } = data + this.buildInDecoder = decoder + } else { + this.buildInDecoder = [] + } + }, + /** * save preferences to local * @returns {Promise}