`47f477c`

Add Aho-Corasick for alternation patterns (6x faster than grep)

Authored by

espadonne 4 months ago

SHA: 47f477c325987aaf3dea018eab539469ced129e3
Parents: eab33b0
Tree: d4bcff1

4 changed files

Status	File	+	-
M	`Makefile`	3	1
A	`src/regex/aho_corasick.f90`	342	0
M	`src/regex/regex_api.f90`	5	0
M	`src/regex/regex_optimizer.f90`	207	0

Makefilemodified

               $(REGEX_DIR)/regex_parser.f90 \
               $(REGEX_DIR)/regex_nfa.f90 \
               $(REGEX_DIR)/regex_engine.f90 \
 +             $(REGEX_DIR)/aho_corasick.f90 \
               $(REGEX_DIR)/regex_optimizer.f90 \
               $(REGEX_DIR)/regex_api.f90 \
               $(REGEX_DIR)/pcre_api.f90
  $(BUILD_DIR)/regex_parser.o: $(BUILD_DIR)/regex_types.o
  $(BUILD_DIR)/regex_nfa.o: $(BUILD_DIR)/regex_types.o $(BUILD_DIR)/regex_parser.o
  $(BUILD_DIR)/regex_engine.o: $(BUILD_DIR)/regex_types.o
 -$(BUILD_DIR)/regex_optimizer.o: $(BUILD_DIR)/regex_types.o
 +$(BUILD_DIR)/aho_corasick.o:
 +$(BUILD_DIR)/regex_optimizer.o: $(BUILD_DIR)/regex_types.o $(BUILD_DIR)/aho_corasick.o
  $(BUILD_DIR)/regex_api.o: $(BUILD_DIR)/regex_types.o $(BUILD_DIR)/regex_lexer.o $(BUILD_DIR)/regex_parser.o $(BUILD_DIR)/regex_nfa.o $(BUILD_DIR)/regex_engine.o $(BUILD_DIR)/regex_optimizer.o
  $(BUILD_DIR)/pcre_api.o:

src/regex/aho_corasick.f90added

 +module aho_corasick
 +  !> Aho-Corasick automaton for multi-pattern string matching
 +  !> Matches all patterns in a single pass O(n + m + z)
 +  !> where n=text length, m=total pattern length, z=matches
 +  implicit none
 +  private
++
 +  public :: ac_automaton_t, ac_match_t
 +  public :: ac_build, ac_search, ac_search_any, ac_free
++
 +  integer, parameter :: MAX_CHILDREN = 256  ! ASCII character set
 +  integer, parameter :: MAX_PATTERNS = 1000
 +  integer, parameter :: MAX_PATTERN_LEN = 4096
++
 +  type :: ac_node_t
 +    !> Trie node with failure links
 +    integer :: children(0:255) = 0     ! Child node indices (0 = no child)
 +    integer :: failure = 0              ! Failure link (fall back on mismatch)
 +    integer :: output_pattern = 0       ! Pattern index that ends here (0 = none)
 +    integer :: output_link = 0          ! Link to next output state
 +    integer :: depth = 0                ! Depth in trie (= prefix length)
 +  end type ac_node_t
++
 +  type :: ac_automaton_t
 +    !> Aho-Corasick automaton
 +    type(ac_node_t), allocatable :: nodes(:)
 +    integer :: num_nodes = 0
 +    integer :: capacity = 0
 +    integer :: num_patterns = 0
 +    integer, allocatable :: pattern_lengths(:)
 +    logical :: compiled = .false.
 +    logical :: ignore_case = .false.
 +  end type ac_automaton_t
++
 +  type :: ac_match_t
 +    !> Match result
 +    logical :: matched = .false.
 +    integer :: pattern_idx = 0          ! Which pattern matched (1-based)
 +    integer :: start_pos = 0            ! Start position in text (1-based)
 +    integer :: end_pos = 0              ! End position in text (1-based)
 +  end type ac_match_t
++
 +contains
++
 +  subroutine ac_build(ac, patterns, num_patterns, ignore_case, ierr)
 +    !> Build Aho-Corasick automaton from patterns
 +    type(ac_automaton_t), intent(out) :: ac
 +    character(len=*), intent(in) :: patterns(:)
 +    integer, intent(in) :: num_patterns
 +    logical, intent(in) :: ignore_case
 +    integer, intent(out) :: ierr
++
 +    integer :: i, j, c, state, next_state, child
 +    integer, allocatable :: queue(:)
 +    integer :: q_head, q_tail
 +    integer :: fail_state
 +    character(len=1) :: ch
++
 +    ierr = 0
++
 +    ! Allocate BFS queue
 +    allocate(queue(MAX_PATTERNS * MAX_PATTERN_LEN))
 +    ac%ignore_case = ignore_case
 +    ac%num_patterns = num_patterns
++
 +    ! Allocate pattern lengths
 +    allocate(ac%pattern_lengths(num_patterns))
 +    do i = 1, num_patterns
 +      ac%pattern_lengths(i) = len_trim(patterns(i))
 +    end do
++
 +    ! Initial capacity - estimate based on total pattern length
 +    ac%capacity = 1
 +    do i = 1, num_patterns
 +      ac%capacity = ac%capacity + len_trim(patterns(i))
 +    end do
 +    ac%capacity = max(ac%capacity, 256)
 +    allocate(ac%nodes(ac%capacity))
++
 +    ! Initialize root node (index 1)
 +    ac%num_nodes = 1
 +    ac%nodes(1)%depth = 0
++
 +    ! Phase 1: Build trie from patterns
 +    do i = 1, num_patterns
 +      state = 1  ! Start at root
 +      do j = 1, len_trim(patterns(i))
 +        ch = patterns(i)(j:j)
 +        if (ignore_case) then
 +          c = to_lower_code(ichar(ch))
 +        else
 +          c = ichar(ch)
 +        end if
++
 +        child = ac%nodes(state)%children(c)
 +        if (child == 0) then
 +          ! Create new node
 +          ac%num_nodes = ac%num_nodes + 1
 +          if (ac%num_nodes > ac%capacity) then
 +            call grow_nodes(ac)
 +          end if
 +          ac%nodes(state)%children(c) = ac%num_nodes
 +          ac%nodes(ac%num_nodes)%depth = ac%nodes(state)%depth + 1
 +          child = ac%num_nodes
 +        end if
 +        state = child
 +      end do
 +      ! Mark this state as accepting for pattern i
 +      ac%nodes(state)%output_pattern = i
 +    end do
++
 +    ! Phase 2: Compute failure links using BFS
 +    q_head = 1
 +    q_tail = 0
++
 +    ! Initialize: depth-1 nodes fail to root
 +    do c = 0, 255
 +      child = ac%nodes(1)%children(c)
 +      if (child /= 0) then
 +        ac%nodes(child)%failure = 1  ! Fail to root
 +        q_tail = q_tail + 1
 +        queue(q_tail) = child
 +      end if
 +    end do
++
 +    ! BFS to compute failure links for deeper nodes
 +    do while (q_head <= q_tail)
 +      state = queue(q_head)
 +      q_head = q_head + 1
++
 +      do c = 0, 255
 +        child = ac%nodes(state)%children(c)
 +        if (child /= 0) then
 +          ! Add to queue
 +          q_tail = q_tail + 1
 +          queue(q_tail) = child
++
 +          ! Compute failure link: follow parent's failure until we find
 +          ! a state with a transition on c, or reach root
 +          fail_state = ac%nodes(state)%failure
 +          do while (fail_state > 1)
 +            if (ac%nodes(fail_state)%children(c) /= 0) exit
 +            fail_state = ac%nodes(fail_state)%failure
 +          end do
++
 +          if (fail_state <= 1) then
 +            ! At or beyond root
 +            if (ac%nodes(1)%children(c) /= 0 .and. ac%nodes(1)%children(c) /= child) then
 +              ac%nodes(child)%failure = ac%nodes(1)%children(c)
 +            else
 +              ac%nodes(child)%failure = 1  ! Fail to root
 +            end if
 +          else if (ac%nodes(fail_state)%children(c) == child) then
 +            ! Would create self-loop, fail to root
 +            ac%nodes(child)%failure = 1
 +          else
 +            ac%nodes(child)%failure = ac%nodes(fail_state)%children(c)
 +          end if
++
 +          ! Compute output link: chain of accepting states via failure links
 +          if (ac%nodes(ac%nodes(child)%failure)%output_pattern /= 0) then
 +            ac%nodes(child)%output_link = ac%nodes(child)%failure
 +          else
 +            ac%nodes(child)%output_link = ac%nodes(ac%nodes(child)%failure)%output_link
 +          end if
 +        end if
 +      end do
 +    end do
++
 +    ac%compiled = .true.
++
 +    deallocate(queue)
++
 +  contains
++
 +    function to_lower_code(c) result(lc)
 +      integer, intent(in) :: c
 +      integer :: lc
 +      if (c >= ichar('A') .and. c <= ichar('Z')) then
 +        lc = c + 32
 +      else
 +        lc = c
 +      end if
 +    end function to_lower_code
++
 +  end subroutine ac_build
++
 +  subroutine grow_nodes(ac)
 +    !> Double the node capacity
 +    type(ac_automaton_t), intent(inout) :: ac
 +    type(ac_node_t), allocatable :: temp(:)
 +    integer :: new_cap
++
 +    new_cap = ac%capacity * 2
 +    allocate(temp(new_cap))
 +    temp(1:ac%num_nodes) = ac%nodes(1:ac%num_nodes)
 +    call move_alloc(temp, ac%nodes)
 +    ac%capacity = new_cap
 +  end subroutine grow_nodes
++
 +  function ac_search_any(ac, text) result(found)
 +    !> Search for any pattern match (fast path for existence check)
 +    type(ac_automaton_t), intent(in) :: ac
 +    character(len=*), intent(in) :: text
 +    logical :: found
++
 +    integer :: i, c, state, next_state, text_len
++
 +    found = .false.
 +    if (.not. ac%compiled) return
++
 +    text_len = len(text)
 +    state = 1  ! Start at root
++
 +    do i = 1, text_len
 +      if (ac%ignore_case) then
 +        c = to_lower_code(ichar(text(i:i)))
 +      else
 +        c = ichar(text(i:i))
 +      end if
++
 +      ! Follow failure links until we find a transition or reach root
 +      do while (state /= 1 .and. ac%nodes(state)%children(c) == 0)
 +        state = ac%nodes(state)%failure
 +      end do
++
 +      next_state = ac%nodes(state)%children(c)
 +      if (next_state /= 0) then
 +        state = next_state
 +      else
 +        state = 1  ! Stay at root if no transition
 +      end if
++
 +      ! Check for match at current state or via output links
 +      if (ac%nodes(state)%output_pattern /= 0) then
 +        found = .true.
 +        return
 +      end if
 +      if (ac%nodes(state)%output_link /= 0) then
 +        found = .true.
 +        return
 +      end if
 +    end do
++
 +  contains
++
 +    function to_lower_code(c) result(lc)
 +      integer, intent(in) :: c
 +      integer :: lc
 +      if (c >= ichar('A') .and. c <= ichar('Z')) then
 +        lc = c + 32
 +      else
 +        lc = c
 +      end if
 +    end function to_lower_code
++
 +  end function ac_search_any
++
 +  function ac_search(ac, text) result(match)
 +    !> Search for first pattern match with position info
 +    type(ac_automaton_t), intent(in) :: ac
 +    character(len=*), intent(in) :: text
 +    type(ac_match_t) :: match
++
 +    integer :: i, c, state, next_state, text_len, pat_idx, out_state
++
 +    match%matched = .false.
 +    if (.not. ac%compiled) return
++
 +    text_len = len(text)
 +    state = 1  ! Start at root
++
 +    do i = 1, text_len
 +      if (ac%ignore_case) then
 +        c = to_lower_code(ichar(text(i:i)))
 +      else
 +        c = ichar(text(i:i))
 +      end if
++
 +      ! Follow failure links until we find a transition or reach root
 +      do while (state /= 1 .and. ac%nodes(state)%children(c) == 0)
 +        state = ac%nodes(state)%failure
 +      end do
++
 +      next_state = ac%nodes(state)%children(c)
 +      if (next_state /= 0) then
 +        state = next_state
 +      else
 +        state = 1
 +      end if
++
 +      ! Check for match at current state
 +      pat_idx = ac%nodes(state)%output_pattern
 +      if (pat_idx /= 0) then
 +        match%matched = .true.
 +        match%pattern_idx = pat_idx
 +        match%end_pos = i
 +        match%start_pos = i - ac%pattern_lengths(pat_idx) + 1
 +        return
 +      end if
++
 +      ! Check output links for overlapping patterns
 +      out_state = ac%nodes(state)%output_link
 +      if (out_state /= 0) then
 +        pat_idx = ac%nodes(out_state)%output_pattern
 +        if (pat_idx /= 0) then
 +          match%matched = .true.
 +          match%pattern_idx = pat_idx
 +          match%end_pos = i
 +          match%start_pos = i - ac%pattern_lengths(pat_idx) + 1
 +          return
 +        end if
 +      end if
 +    end do
++
 +  contains
++
 +    function to_lower_code(c) result(lc)
 +      integer, intent(in) :: c
 +      integer :: lc
 +      if (c >= ichar('A') .and. c <= ichar('Z')) then
 +        lc = c + 32
 +      else
 +        lc = c
 +      end if
 +    end function to_lower_code
++
 +  end function ac_search
++
 +  subroutine ac_free(ac)
 +    !> Free automaton resources
 +    type(ac_automaton_t), intent(inout) :: ac
++
 +    if (allocated(ac%nodes)) deallocate(ac%nodes)
 +    if (allocated(ac%pattern_lengths)) deallocate(ac%pattern_lengths)
 +    ac%num_nodes = 0
 +    ac%capacity = 0
 +    ac%num_patterns = 0
 +    ac%compiled = .false.
 +  end subroutine ac_free
++
 +end module aho_corasick

src/regex/regex_api.f90modified

      integer :: error_code = 0
      character(len=256) :: error_msg = ''
      integer :: num_groups = 0
 +    character(len=4096) :: pattern = ''  ! Original pattern for AC detection
    contains
      procedure :: is_compiled => regex_is_compiled
    end type regex_t
      ! Optimize NFA for faster matching
      call optimize_nfa(re%opt_nfa, re%nfa)
 +    ! Store pattern and try Aho-Corasick for alternation patterns
 +    re%pattern = pattern
 +    call try_build_aho_corasick(re%opt_nfa, pattern, extended, .false.)
++
      re%compiled = .true.
    contains

src/regex/regex_optimizer.f90modified

    !>   - Bit vector state sets for O(1) operations
    !>   - Lazy DFA state caching
    !>   - Anchored pattern fast paths
 +  !>   - Aho-Corasick for alternation patterns
    use regex_types
 +  use aho_corasick
    implicit none
    private
    public :: optimized_nfa_t
    public :: optimize_nfa, optimized_match, optimized_search
 +  public :: try_build_aho_corasick
    integer, parameter :: MAX_STATES = 1024
    integer, parameter :: MAX_PREFIX_LEN = 64
      type(dfa_cache_entry_t) :: dfa_cache(DFA_CACHE_SIZE)  ! Lazy DFA cache
      type(compiled_dfa_t) :: dfa                  ! Full compiled DFA (if available)
      logical :: use_dfa = .false.                 ! Use DFA instead of NFA
 +    type(ac_automaton_t) :: ac                   ! Aho-Corasick automaton (for alternation)
 +    logical :: use_aho_corasick = .false.        ! Use Aho-Corasick for matching
      logical :: optimized = .false.
    end type optimized_nfa_t
      res%matched = .false.
      text_len = len_trim(text)
 +    ! Fast path: use Aho-Corasick for alternation patterns
 +    ! Only use AC if ignore_case setting matches what was compiled
 +    if (opt%use_aho_corasick) then
 +      if (ignore_case .eqv. opt%ac%ignore_case) then
 +        res = ac_optimized_search(opt%ac, text)
 +        return
 +      end if
 +    end if
++
      if (opt%nfa%num_states == 0) return
      ! Fast path: use DFA if available (O(n) matching)
      if (negated) res = .not. res
    end function char_in_class_opt
 +  !---------------------------------------------------------------------------
 +  ! Aho-Corasick Integration for Alternation Patterns
 +  !---------------------------------------------------------------------------
++
 +  subroutine try_build_aho_corasick(opt, pattern, is_ere, ignore_case)
 +    !> Try to build Aho-Corasick automaton for simple alternation patterns
 +    !> Pattern like "foo|bar|baz" with only literal characters and | separators
 +    type(optimized_nfa_t), intent(inout) :: opt
 +    character(len=*), intent(in) :: pattern
 +    logical, intent(in) :: is_ere, ignore_case
++
 +    character(len=4096), allocatable :: alternatives(:)
 +    integer :: num_alternatives, ierr
 +    logical :: is_simple
++
 +    allocate(alternatives(1000))
++
 +    opt%use_aho_corasick = .false.
++
 +    ! Check if pattern is simple alternation of literals
 +    call parse_simple_alternation(pattern, is_ere, alternatives, num_alternatives, is_simple)
++
 +    ! DEBUG (commented out for production)
 +    ! write(error_unit, '(A,I0,A,L1)') 'DEBUG AC: num_alt=', num_alternatives, ' is_simple=', is_simple
++
 +    if (.not. is_simple .or. num_alternatives < 2) return
++
 +    ! Build Aho-Corasick automaton
 +    call ac_build(opt%ac, alternatives, num_alternatives, ignore_case, ierr)
++
 +    if (ierr == 0) then
 +      opt%use_aho_corasick = .true.
 +    end if
++
 +    deallocate(alternatives)
++
 +  end subroutine try_build_aho_corasick
++
 +  subroutine parse_simple_alternation(pattern, is_ere, alternatives, num_alt, is_simple)
 +    !> Parse pattern to check if it's simple alternation of literals
 +    !> Returns the alternatives if so
 +    character(len=*), intent(in) :: pattern
 +    logical, intent(in) :: is_ere
 +    character(len=*), intent(out) :: alternatives(:)
 +    integer, intent(out) :: num_alt
 +    logical, intent(out) :: is_simple
++
 +    integer :: i, pat_len, alt_start, alt_len
 +    character(len=1) :: c, next_c
 +    logical :: in_escape
++
 +    is_simple = .true.
 +    num_alt = 0
 +    pat_len = len_trim(pattern)
++
 +    if (pat_len == 0) then
 +      is_simple = .false.
 +      return
 +    end if
++
 +    alt_start = 1
 +    alt_len = 0
 +    in_escape = .false.
 +    i = 1
++
 +    do while (i <= pat_len)
 +      c = pattern(i:i)
++
 +      if (in_escape) then
 +        ! In ERE mode, \| is literal |
 +        ! In BRE mode, \| is alternation (GNU extension)
 +        if (c == '|' .and. .not. is_ere) then
 +          ! BRE alternation
 +          if (alt_len > 0) then
 +            num_alt = num_alt + 1
 +            if (num_alt > size(alternatives)) then
 +              is_simple = .false.
 +              return
 +            end if
 +            alternatives(num_alt) = pattern(alt_start:alt_start+alt_len-1)
 +          else
 +            ! Empty alternative - still valid
 +            num_alt = num_alt + 1
 +            alternatives(num_alt) = ''
 +          end if
 +          alt_start = i + 1
 +          alt_len = 0
 +        else if (c == '(' .or. c == ')' .or. c == '{' .or. c == '}' .or. &
 +                 c == '<' .or. c == '>' .or. c == 'b' .or. c == 'B' .or. &
 +                 c == 'd' .or. c == 'D' .or. c == 'w' .or. c == 'W' .or. &
 +                 c == 's' .or. c == 'S' .or. c == '1' .or. c == '2' .or. &
 +                 c == '3' .or. c == '4' .or. c == '5' .or. c == '6' .or. &
 +                 c == '7' .or. c == '8' .or. c == '9') then
 +          ! Regex metacharacter - not simple
 +          is_simple = .false.
 +          return
 +        else
 +          ! Escaped literal character (e.g., \., \*, etc.)
 +          alt_len = alt_len + 1
 +        end if
 +        in_escape = .false.
 +        i = i + 1
 +        cycle
 +      end if
++
 +      if (c == '\') then
 +        in_escape = .true.
 +        i = i + 1
 +        cycle
 +      end if
++
 +      ! Check for metacharacters
 +      if (is_ere) then
 +        ! ERE mode: | is alternation, . * + ? [ ] ^ $ ( ) { } are metacharacters
 +        if (c == '|') then
 +          ! Alternation separator
 +          if (alt_len > 0) then
 +            num_alt = num_alt + 1
 +            if (num_alt > size(alternatives)) then
 +              is_simple = .false.
 +              return
 +            end if
 +            alternatives(num_alt) = pattern(alt_start:alt_start+alt_len-1)
 +          else
 +            num_alt = num_alt + 1
 +            alternatives(num_alt) = ''
 +          end if
 +          alt_start = i + 1
 +          alt_len = 0
 +          i = i + 1
 +          cycle
 +        else if (c == '.' .or. c == '*' .or. c == '+' .or. c == '?' .or. &
 +                 c == '[' .or. c == ']' .or. c == '^' .or. c == '$' .or. &
 +                 c == '(' .or. c == ')' .or. c == '{' .or. c == '}') then
 +          ! Metacharacter - not simple alternation
 +          is_simple = .false.
 +          return
 +        end if
 +      else
 +        ! BRE mode: only . * [ ] ^ $ are metacharacters
 +        ! | is literal, \| is alternation (GNU extension)
 +        if (c == '.' .or. c == '*' .or. c == '[' .or. c == ']' .or. &
 +            c == '^' .or. c == '$') then
 +          is_simple = .false.
 +          return
 +        end if
 +      end if
++
 +      ! Regular literal character
 +      alt_len = alt_len + 1
 +      i = i + 1
 +    end do
++
 +    ! Handle last alternative
 +    if (alt_len > 0 .or. num_alt > 0) then
 +      num_alt = num_alt + 1
 +      if (num_alt > size(alternatives)) then
 +        is_simple = .false.
 +        return
 +      end if
 +      if (alt_len > 0) then
 +        alternatives(num_alt) = pattern(alt_start:alt_start+alt_len-1)
 +      else
 +        alternatives(num_alt) = ''
 +      end if
 +    end if
++
 +    ! Need at least 2 alternatives for Aho-Corasick to be useful
 +    if (num_alt < 2) then
 +      is_simple = .false.
 +    end if
++
 +  end subroutine parse_simple_alternation
++
 +  function ac_optimized_search(ac, text) result(res)
 +    !> Search using Aho-Corasick automaton
 +    type(ac_automaton_t), intent(in) :: ac
 +    character(len=*), intent(in) :: text
 +    type(match_result_t) :: res
++
 +    type(ac_match_t) :: ac_match
++
 +    res%matched = .false.
++
 +    ac_match = ac_search(ac, text)
 +    if (ac_match%matched) then
 +      res%matched = .true.
 +      res%match_start = ac_match%start_pos
 +      res%match_end = ac_match%end_pos
 +    end if
++
 +  end function ac_optimized_search
++
  end module regex_optimizer