网络入侵检测系统之 Suricata(四)-- 初始化模块代码详解

2022 年 9 月 17 日
江苏
本文字数：9422 字
阅读完需：约 31 分钟

Initial Module 初始化流程

初始化 Suricata instance 用来保存程序当前的一些状态、标志等上下文环境，通常是用来作为参数传递给各个模块的子函数 memset(suri, 0x00, sizeof(suri));// pointer to argv[0]suri->progname = progname;//运行模式 suri->run_mode = RUNMODE_UNKNOWN;memset(suri->pcap_dev, 0, sizeof(suri->pcap_dev));//签名文件 suri->sig_file = NULL;suri->sig_file_exclusive = FALSE;suri->pid_filename = NULL;//正则参数 suri->regex_arg = NULL;//延迟检测开关 suri->delayed_detect = 0;//守护进程 suri->daemon = 0;//离线模式 suri->offline = 0;//详细模式 suri->verbose = 0;/ use -1 as unknown */suri->checksum_validation = -1;//是否检测 g_detect_disabled = suri->disabled_detect = 1;2. 初始化原子变量 engine_stage –> 记录程序当前的运行阶段：SURICATA_INIT、SURICATA_RUNTIME、SURICATA_FINALIZE

suricata_context 初始化//打印相关 console or file 以及 levelsuricata_context.SCLogMessage = SCLogMessage;// 文件 io 相关的函数 suricata_context.FileOpenFileWithId = FileOpenFileWithId;suricata_context.FileCloseFileById = FileCloseFileById;suricata_context.FileAppendDataById = FileAppendDataById;suricata_context.FileAppendGAPById = FileAppendGAPById;suricata_context.FileContainerRecycle = FileContainerRecycle;suricata_context.FilePrune = FilePrune;suricata_context.FileSetTx = FileContainerSetTx;3. 以环境变量或缺省值对日志模块初始化

op_iface 日志输出接口：console or file or syslog 文件指针文件锁日志格式和 levelop_filter_regex 只输出匹配到这个正则的文件名 4. 设置当前主线程名字为“Suricata-Main”。在 gdb 调试时 info threads 可以看到各个线程名

初始化 ParserSize 模块，正则预编译，用于解析类似“10Mb”这种大小参数

#define PARSE_REGEX "^\s*(\d+(?:.\d+)?)\s*([a-zA-Z]{2})?\s*$"6. 注册各种运行模式。Suricata 对“运行模式”这个概念也进行了封装。运行模式存储在 runmodes 数组中，定义为 RunModes runmodes[RUNMODE_USER_MAX]

enum RunModes {RUNMODE_UNKNOWN = 0,RUNMODE_PCAP_DEV,RUNMODE_PCAP_FILE,RUNMODE_PFRING,RUNMODE_NFQ,RUNMODE_NFLOG,RUNMODE_IPFW,RUNMODE_ERF_FILE,RUNMODE_DAG,RUNMODE_AFP_DEV,RUNMODE_NETMAP,RUNMODE_UNITTEST,RUNMODE_NAPATECH,RUNMODE_UNIX_SOCKET,RUNMODE_WINDIVERT,RUNMODE_PLUGIN,}typedef struct RunModes_ {int cnt;RunMode *runmodes;} RunModes;7. 注册各种运行模式以 IDS Pcap 运行模式为例，它由 3 个子运行模式组成，底层由 realloc 分配空间，IDS Pcap Mode =RunModeIdsPcapSingle（single thread pcap processing）+ RunModeIdsPcapAutoFp（默认子运行模式）+RunModeIdsPcapWorkers（Workers version of the PCAP LIVE processing）等后续初始化阶段确定了具体的运行模式后，就会调用这 3 个注册的对应的初始化函数，对该模式下的运行环境进行进一步配置

/*当前子模式：RunMode *mode = &runmodes[runmode].runmodes[runmodes[runmode].cnt];*/RunModeIdsPcapRegister();RunModeFilePcapRegister();RunModeIdsPfringRegister();RunModeIpsNFQRegister();RunModeIpsIPFWRegister();RunModeErfFileRegister();RunModeErfDagRegister();RunModeNapatechRegister();RunModeIdsAFPRegister();RunModeIdsNetmapRegister();RunModeIdsNflogRegister();RunModeUnixSocketRegister();RunModeIpsWinDivertRegister();8. 初始化配置模块，为后续解析 Suricata.yaml 建立配置节点树的根节点 root，底层调用为 alloc

typedef struct ConfNode_ {//filename: fast.log name:valchar *name;char *val;/**< Flag that sets this nodes value as final. */int final;//父节点 struct ConfNode_ *parent;//头节点，next 节点 TAILQ_HEAD(, ConfNode_) head;TAILQ_ENTRY(ConfNode_) next;} ConfNode;9. 命令行参数解析

//监控设备注册 typedef struct LiveDeviceName_ {char *dev; /**< the device (e.g. "eth0") */TAILQ_ENTRY(LiveDeviceName_) next;} LiveDeviceName;pd = SCCalloc(1, sizeof(LiveDeviceName));TAILQ_INSERT_TAIL(&pre_live_devices, pd, next);g_engine_mode 默认为 IDS，IPS 是可以 drop 包(--simulate-ips)EngineModeSetIPS();短命令行参数解析，并添加到 Conf Tree 中：ConfGetNodeOrCreate + setVal

“-v”选项可多次使用，每个 v 都能将当前日志等级提升一级 suri->verbose++;设置全局的 run_mode 变量，校验 daemon 模式是否兼容 run_mode（Pcap 文件模式及单元测试模式都不能在 daemon 开启下进行）10. GlobalsInitPreConfig 获取当前时间所用的 spin lock，以及设置时区（调用 tzset()即可）多模快速匹配链表维护，按优先级排序 sm_fp_support_smlist_list，底层 SCMalloc 匹配阈值关键字相关，正则预编译

LoadYamlConfig 调用 LoadYamlConfig 读取 Yaml 格式配置文件。Yaml 格式解析是通过 libyaml 库来完成的，解析的结果存储在配置节点树（见 conf.c）中
SCLogLoadConfig 根据之前转化好的配置树，初始化日志模块一些全局配置

SCLogDebug("sc_log_global_log_level: %d", sc_log_global_log_level);SCLogDebug("sc_lc->log_format: %s", sc_log_config->log_format);SCLogDebug("SCLogSetOPFilter: filter: %s", sc_log_config->op_filter);typedef struct SCLogConfig_{

char *startup_message;

SCLogLevel log_level;

char *log_format;

char op_filter;/ compiled pcre filter expression */

pcre *op_filter_regex;

pcre_extra op_filter_regex_study;/ op ifaces used */

SCLogOPIfaceCtx op_ifaces;/ no of op ifaces */

uint8_t op_ifaces_cnt;} SCLogConfig;13. PostConfLoadedSetup 配置生效后，模块 setup

MpmTableSetup：设置多模式匹配表，该表中每一项就是一个实现了某种多模式匹配算法（如 WuManber、AC）的匹配器 ac,ac-ks,ac-bs,hsSpmTableSetup:设置单模匹配表， bm,hstypedef struct MpmTableElmt_ {const char *name;void (*InitCtx)(struct MpmCtx_ *);void (*InitThreadCtx)(struct MpmCtx_ *, struct MpmThreadCtx_ *);void (*DestroyCtx)(struct MpmCtx_ *);void (*DestroyThreadCtx)(struct MpmCtx_ *, struct MpmThreadCtx_ *);

/** function pointers for adding patterns to the mpm ctx. * *  \param mpm_ctx Mpm context to add the pattern to *  \param pattern pointer to the pattern *  \param pattern_len length of the pattern in bytes *  \param offset pattern offset setting *  \param depth pattern depth setting *  \param pid pattern id *  \param sid signature _internal_ id *  \param flags pattern flags */int  (*AddPattern)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);int  (*AddPatternNocase)(struct MpmCtx_ *, uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t);int  (*Prepare)(struct MpmCtx_ *);uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t);void (*PrintCtx)(struct MpmCtx_ *);void (*PrintThreadCtx)(struct MpmThreadCtx_ *);void (*RegisterUnittests)(void);uint8_t flags;

复制代码

} MpmTableElmt;

extern MpmTableElmt mpm_table[MPM_TABLE_SIZE];

typedef struct MpmCtx_ {void *ctx;uint8_t mpm_type;

uint8_t flags;
uint16_t maxdepth;
/* unique patterns */uint32_t pattern_cnt;
uint16_t minlen;uint16_t maxlen;
uint32_t memory_cnt;uint32_t memory_size;
uint32_t max_pat_id;
/* hash used during ctx initialization */MpmPattern **init_hash;

复制代码

} MpmCtx;

typedef struct SCACCtx_ {/* pattern arrays. We need this only during the goto table creation phase */MpmPattern **parray;

/* no of states used by ac */uint32_t state_count;
uint32_t pattern_id_bitarray_size;
/* the all important memory hungry state_table */SC_AC_STATE_TYPE_U16 (*state_table_u16)[256];/* the all important memory hungry state_table */SC_AC_STATE_TYPE_U32 (*state_table_u32)[256];
/* goto_table, failure table and output table.  Needed to create state_table. * Will be freed, once we have created the state_table */int32_t (*goto_table)[256];int32_t *failure_table;SCACOutputTable *output_table;SCACPatternList *pid_pat_list;
/* the size of each state */uint32_t single_state_size;
uint32_t allocated_state_count;

复制代码

} SCACCtx;存储相关初始化如 tag 关键字，由于这类信息依赖于规则，同时出现的频率不高,申请内存 STORAGE_HOST,STORAGE_FLOW,STORAGE_IPPAIR,STORAGE_DEVICEmemset(&storage_max_id, 0x00, sizeof(storage_max_id));storage_list = NULL;storage_map = NULL;storage_registraton_closed = 0;

注册 RegisterFlowBypassInfo 旁路流量卸载 STORAGE_FLOW 协议解析器注册 AppLayerParserProtoCtx ctxs[FLOW_PROTO_MAX][ALPROTO_MAX];enum {FLOW_PROTO_TCP = 0,FLOW_PROTO_UDP,FLOW_PROTO_ICMP,FLOW_PROTO_DEFAULT,

/* should be last */FLOW_PROTO_MAX,

复制代码

};enum AppProtoEnum {ALPROTO_UNKNOWN = 0,ALPROTO_HTTP,ALPROTO_FTP,ALPROTO_SMTP,ALPROTO_TLS, /* SSLv2, SSLv3 & TLSv1 */ALPROTO_SSH,ALPROTO_IMAP,ALPROTO_JABBER,ALPROTO_SMB,ALPROTO_DCERPC,ALPROTO_IRC,

ALPROTO_DNS,ALPROTO_MODBUS,ALPROTO_ENIP,ALPROTO_DNP3,ALPROTO_NFS,ALPROTO_NTP,ALPROTO_FTPDATA,ALPROTO_TFTP,ALPROTO_IKEV2,ALPROTO_KRB5,ALPROTO_DHCP,ALPROTO_SNMP,ALPROTO_SIP,ALPROTO_RFB,ALPROTO_MQTT,ALPROTO_TEMPLATE,ALPROTO_TEMPLATE_RUST,ALPROTO_RDP,ALPROTO_HTTP2,
/* used by the probing parser when alproto detection fails * permanently for that particular stream */ALPROTO_FAILED,

复制代码

#ifdef UNITTESTSALPROTO_TEST,#endif /* UNITESTS // keep last */ALPROTO_MAX,};

/**

\brief App layer protocol parser context./typedef struct AppLayerParserProtoCtx_{/ 0 - to_server, 1 - to_client. */AppLayerParserFPtr Parser[2];bool logger;uint32_t logger_bits; /**< registered loggers for this proto */
void *(*StateAlloc)(void *, AppProto);void (*StateFree)(void *);void (*StateTransactionFree)(void *, uint64_t);void *(*LocalStorageAlloc)(void);void (*LocalStorageFree)(void *);
void (*Truncate)(void *, uint8_t);FileContainer *(*StateGetFiles)(void *, uint8_t);AppLayerDecoderEvents *(*StateGetEvents)(void *);
int (*StateGetProgress)(void *alstate, uint8_t direction);uint64_t (*StateGetTxCnt)(void *alstate);void *(*StateGetTx)(void *alstate, uint64_t tx_id);AppLayerGetTxIteratorFunc StateGetTxIterator;int (*StateGetProgressCompletionStatus)(uint8_t direction);int (*StateGetEventInfoById)(int event_id, const char **event_name,AppLayerEventType *event_type);int (*StateGetEventInfo)(const char *event_name,int *event_id, AppLayerEventType *event_type);
DetectEngineState *(*GetTxDetectState)(void *tx);int (*SetTxDetectState)(void *tx, DetectEngineState *);
AppLayerTxData *(*GetTxData)(void *tx);bool (*ApplyTxConfig)(void *state, void *tx, int mode, AppLayerTxConfig);
void (*SetStreamDepthFlag)(void *tx, uint8_t flags);
/* each app-layer has its own value */uint32_t stream_depth;
/* Indicates the direction the parser is ready to see the data
the first time for a flow. Values accepted -
STREAM_TOSERVER, STREAM_TOCLIENT */uint8_t first_data_dir;
/* Option flags such as supporting gaps or not. /uint32_t option_flags;/ coccinelle: AppLayerParserProtoCtx:option_flags:APP_LAYER_PARSER_OPT_ */
uint32_t internal_flags;/* coccinelle: AppLayerParserProtoCtx:internal_flags:APP_LAYER_PARSER_INT_ */

#ifdef UNITTESTSvoid (*RegisterUnittests)(void);#endif} AppLayerParserProtoCtx;

http 解析器的 HTTP state memorytypedef struct HtpState_ {/* Connection parser structure for each connection */htp_connp_t connp;/ Connection structure for each connection */htp_conn_t *conn;Flow *f; /< Needed to retrieve the original flow when using HTPLib callbacks */uint64_t transaction_cnt;uint64_t store_tx_id;FileContainer *files_ts;FileContainer *files_tc;const struct HTPCfgRec_ *cfg;uint16_t flags;uint16_t events;uint16_t htp_messages_offset; /< offset into conn->messages list */uint32_t file_track_id; /**< used to assign file track ids to files */uint64_t last_request_data_stamp;uint64_t last_response_data_stamp;} HtpState;14. SCHInfoLoadFromConfig：从配置文件中载入 host os policy(主机 OS 策略)信息 radix tree 保存所有主机策略信息

SigTableSetup：初始化检测引擎，主要是注册检测引擎所支持的规则格式（跟 Snort 规则基本一致）中的关键字，比如 sid、priority、msg、within、distance 等等

typedef struct SigTableElmt_ {/** Packet match function pointer */int (*Match)(DetectEngineThreadCtx *, Packet *, const Signature *, const SigMatchCtx *);

/** AppLayer TX match function pointer */int (*AppLayerTxMatch)(DetectEngineThreadCtx *, Flow *,        uint8_t flags, void *alstate, void *txv,        const Signature *, const SigMatchCtx *);
/** File match function  pointer */int (*FileMatch)(DetectEngineThreadCtx *,    Flow *,                     /**< *LOCKED* flow */    uint8_t flags, File *, const Signature *, const SigMatchCtx *);
/** InspectionBuffer transformation callback */void (*Transform)(InspectionBuffer *, void *context);bool (*TransformValidate)(const uint8_t *content, uint16_t content_len, void *context);
/** keyword setup function pointer */int (*Setup)(DetectEngineCtx *, Signature *, const char *);
bool (*SupportsPrefilter)(const Signature *s);int (*SetupPrefilter)(DetectEngineCtx *de_ctx, struct SigGroupHead_ *sgh);
void (*Free)(DetectEngineCtx *, void *);

复制代码

#ifdef UNITTESTSvoid (RegisterTests)(void);#endifuint16_t flags;/ coccinelle: SigTableElmt:flags:SIGMATCH_ */

/** better keyword to replace the current one */uint16_t alternative;
const char *name;     /**< keyword name alias */const char *alias;    /**< name alias */const char *desc;const char *url;

复制代码

} SigTableElmt;

规则结构体

/** \brief Signature container /typedef struct Signature_ {uint32_t flags;/ coccinelle: Signature:flags:SIG_FLAG_ */

AppProto alproto;
uint16_t dsize_low;uint16_t dsize_high;
SignatureMask mask;SigIntId num; /**< signature number, internal id */
/** inline -- action */uint8_t action;uint8_t file_flags;
/** addresses, ports and proto this sig matches on */DetectProto proto;
/** classification id **/uint16_t class_id;
/** ipv4 match arrays */uint16_t addr_dst_match4_cnt;uint16_t addr_src_match4_cnt;uint16_t addr_dst_match6_cnt;uint16_t addr_src_match6_cnt;DetectMatchAddressIPv4 *addr_dst_match4;DetectMatchAddressIPv4 *addr_src_match4;/** ipv6 match arrays */DetectMatchAddressIPv6 *addr_dst_match6;DetectMatchAddressIPv6 *addr_src_match6;
uint32_t id;  /**< sid, set by the 'sid' rule keyword */uint32_t gid; /**< generator id */uint32_t rev;int prio;
/** port settings for this signature */DetectPort *sp, *dp;

复制代码

#ifdef PROFILINGuint16_t profiling_id;#endif

/** netblocks and hosts specified at the sid, in CIDR format */IPOnlyCIDRItem *CidrSrc, *CidrDst;
DetectEngineAppInspectionEngine *app_inspect;DetectEnginePktInspectionEngine *pkt_inspect;
/* Matching structures for the built-ins. The others are in * their inspect engines. */SigMatchData *sm_arrays[DETECT_SM_LIST_MAX];
/* memory is still owned by the sm_lists/sm_arrays entry */const struct DetectFilestoreData_ *filestore_ctx;
char *msg;
/** classification message */char *class_msg;/** Reference */DetectReference *references;/** Metadata */DetectMetadataHead *metadata;
char *sig_str;
SignatureInitData *init_data;
/** ptr to the next sig in the list */struct Signature_ *next;

复制代码

} Signature;17. TmqhSetup：Tm-queue 是各个模块（线程）之间传递数据的缓冲区初始化 queue handler（队列处理函数），这个是衔接线程模块和数据包队列之间的桥梁，目前共有 5 类 handler：simple, nfq, packetpool, flow, ringbuffer。每类 handler 内部都有一个 InHandler 和 OutHandler，一个用于从上一级队列中获取数据包，另一个用于处理完毕后将数据包送入下一级队列。

Packepool 通过 read 和 write 两个位置标记对 packetpool(ringbuffer)这个循环队列进行进出操作。
Simple 按照 FIFO（先进先出）原则对缓冲区内容进行进出操作。
Flow 出队的时候是按照 FIFO 进行，入队的时候对数据包的头部信息进行 hash，然后将具有相同 hash 值的数据包放到一个缓冲区。Tmqh tmqh_table[TMQH_SIZE];

RegisterAllModules：注册所有模块里面注册了 Suricata 所支持的所有线程模块（Thread Module）。以 pcap 相关模块为例，TmModuleReceivePcapRegister 函数注册了 Pcap 捕获模块，而 TmModuleDecodePcapRegister 函数注册了 Pcap 数据包解码模块。所谓注册，就是在 tmm_modules 模块数组中对应的那项中填充 TmModule 结构的所有字段，这些字段包括：模块名字、线程初始化函数、包处理或包获取函数、线程退出清理函数、一些标志位等等。

typedef struct TmModule_ {const char *name;

/** thread handling */TmEcode (*ThreadInit)(ThreadVars *, const void *, void **);void (*ThreadExitPrintStats)(ThreadVars *, void *);TmEcode (*ThreadDeinit)(ThreadVars *, void *);
/** the packet processing function */TmEcode (*Func)(ThreadVars *, Packet *, void *);
TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);
/** terminates the capture loop in PktAcqLoop */TmEcode (*PktAcqBreakLoop)(ThreadVars *, void *);
TmEcode (*Management)(ThreadVars *, void *);
/** global Init/DeInit */TmEcode (*Init)(void);TmEcode (*DeInit)(void);

复制代码

#ifdef UNITTESTSvoid (*RegisterTests)(void);#endifuint8_t cap_flags; /**< Flags to indicate the capability requierment ofthe given TmModule // Other flags used by the module */uint8_t flags;} TmModule;

extern TmModule tmm_modules[TMM_SIZE]

typedef enum {TMM_FLOWWORKER,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_RECEIVEPLUGIN,TMM_DECODEPLUGIN,TMM_RESPONDREJECT,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,TMM_RECEIVEAFP,TMM_DECODEAFP,TMM_RECEIVENETMAP,TMM_DECODENETMAP,TMM_ALERTPCAPINFO,TMM_RECEIVENAPATECH,TMM_DECODENAPATECH,TMM_STATSLOGGER,TMM_RECEIVENFLOG,TMM_DECODENFLOG,TMM_RECEIVEWINDIVERT,TMM_VERDICTWINDIVERT,TMM_DECODEWINDIVERT,

TMM_FLOWMANAGER,TMM_FLOWRECYCLER,TMM_BYPASSEDFLOWMANAGER,TMM_DETECTLOADER,
TMM_UNIXMANAGER,
TMM_SIZE,

复制代码

} TmmId;19. 规则加载 LoadSignatures-》得到 signutrue listDetectEngineCtx_->sig_list + sig_cnt->排序 ByAction，byFlowbits,，ByFlowvar，ByPktvar，ByHostbits，ByIPPairbit，ByPriority->runtime match structure （siggroup），siggroup 的过程目前看是把之前的所有规则 list 通过 ip，端口号和流分组到一个链表数组（sgh，并维护了一个 sigMask(规则有哪些匹配项)

$25 = {flags = 1572879, alproto = 0, dsize_low = 0, dsize_high = 0, mask = 0 '\000',       num = 0, action = 1 '\001', file_flags = 0 '\000', proto = {proto = '\377'        <repeats 32 times>, flags = 1 '\001'}, class = 26 '\032',addr_dst_match4_cnt = 1,        addr_src_match4_cnt = 1, addr_dst_match6_cnt = 1,addr_src_match6_cnt = 1, addr_dst_match4 = 0x1867db0, addr_src_match4 = 0x1867df0,addr_dst_match6 = 0x1867f30, addr_src_match6 = 0x1867f00, id = 2260002, gid = 1,rev = 1, prio = 3, sp = 0x1867c70, dp = 0x1867ca0, CidrSrc = 0x1867bb0,CidrDst = 0x1867c40, app_inspect = 0x0, sm_arrays = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,0x0}, filestore_ctx = 0x0,msg = 0x1867cd0 "SURICATA Applayer Detect protocol only one direction",class_msg = 0x1855880 "Generic Protocol Command Decode", references = 0x0,metadata = 0x0,sig_str = 0x1867980 "alert ip any any -> any any (msg:"SURICATA Applayer Detect protocol only one direction"; flow:established; app-layer-event:applayer_detect_protocol_only_one_direction; flowint:applayer.anomaly.count,+"..., init_data = 0x1856830,next = 0x0}(gdb)
/* Signature flags //* \note: additions should be added to the rule analyzer as well */
#define SIG_FLAG_SRC_ANY                BIT_U32(0) /< source is any */    #define SIG_FLAG_DST_ANY                BIT_U32(1)  /< destination is any */#define SIG_FLAG_SP_ANY                 BIT_U32(2)  /< source port is any */#define SIG_FLAG_DP_ANY                 BIT_U32(3)  /< destination port is any */
#define SIG_FLAG_NOALERT                BIT_U32(4)  /< no alert flag is set */#define SIG_FLAG_DSIZE                  BIT_U32(5)  /< signature has a dsize setting */#define SIG_FLAG_APPLAYER               BIT_U32(6)  /< signature applies to app layer instead of packets */#define SIG_FLAG_IPONLY                 BIT_U32(7)  /< ip only signature */