#!/usr/bin/perl -w # # @(#)$Id: bnf2yacc.pl,v 1.16 2017/11/14 06:53:22 jleffler Exp $ # # Convert SQL-92, SQL-99 BNF plain text file into YACC grammar. use strict; $| = 1; use constant debug => 0; my $heading = ""; my %tokens; my %nonterminals; my %rules; my %used; my $start; my @grammar; my $nt_number = 0; # Generate a new non-terminal identifier sub new_non_terminal { my($prefix) = @_; $prefix = "" unless defined $prefix; return sprintf "${prefix}nt_%03d", ++$nt_number; } # map_non_terminal converts names that are not acceptable to Yacc into names that are. # Non-identifier characters are converted to underscores. # If the first character is not alphabetic, prefix 'j_'. # Case-convert to lower case. sub map_non_terminal { my($nt) = @_; $nt =~ s/\W+/_/go; $nt = "j_$nt" unless $nt =~ m/^[a-zA-Z]/o; $nt =~ tr/[A-Z]/[a-z]/; $nt =~ s/__+/_/go; return $nt; } # scan_rhs breaks up the RHS of a rule into a token stream # Keywords (terminals) are prefixed with a '#' marker. sub scan_rhs { my($tail) = @_; my(@rhs); while ($tail) { print "RHS: $tail\n" if debug; my $name; if ($tail =~ m%^(\s*<([-:/()_\w\s]+)>\s*)%o) { # Simpler regex for non-terminal: <[^>]+> # Non-terminal my $n = $2; print "N: $n\n" if debug; $tail = substr $tail, length($1); $name = map_non_terminal($n); $nonterminals{$name} = 1; $used{$name} = 1; push @rhs, $name; } elsif ($tail =~ m%^(\s*(\w[-\w\d_.]*)\s*)%o) { # Terminal (keyword) # Dot '.' is used in Interfaces.SQL in Ada syntax # Dash '-' is used in EXEC-SQL in the keywords. my $t = $2; print "T: $t\n" if debug; $tail = substr $tail, length($1); $name = $t; $tokens{$name} = 1; push @rhs, "#$name"; } elsif ($tail =~ m%^\s*(\.\.\.omitted\.\.\.)\s*%o) { # Something omitted from the grammar. # Triple punctuation detected before double. my $str = "/* $1 */"; push @rhs, $str; last; } elsif ($tail =~ m{^(\s*([-.<=>|]{2})\s*)$}o) { # Double-punctuation (non-metacharacters) # .., <=, >=, <>, ||, -> my $p = $2; print "DP: $p\n" if debug; $tail = substr $tail, length($1); $name = "'$p'"; push @rhs, $name; } elsif ($tail =~ m{^(\s*([][{}"'%&()*+,-./:;<=>?^_|])\s*)$}o) { # Punctuation (non-metacharacters) # Note that none of '@', '~', '!' or '\' have any significance in SQL my $p = $2; print "P: $p\n" if debug; $tail = substr $tail, length($1); $p = "\\'" if $p eq "'"; $name = "'$p'"; push @rhs, $name; } elsif ($tail =~ m%^(\s*('[^']*'))\s*%o || $tail =~ m%^(\s*("[^"]*"))\s*%o) { # Terminal in quotes - single or double. # (Possibly a multi-character string). my $q = $2; print "Q: $q\n" if debug; $tail = substr $tail, length($1); $q =~ m%^(['"])(.+)['"]$%o; # Expand multi-character string constants. # into repeated single-character constants. my($o) = $1; my($l) = $2; while (length($l)) { my($c) = substr $l, 0, 1; $name = "$o$c$o"; $l = substr $l, 1, length($l)-1; push @rhs, $name; } } elsif ($tail =~ m%^(\s*([{}\|\[\]]|\.\.\.)\s*)%o) { # Punctuation (metacharacters) my $p = $2; print "M: $p\n" if debug; $tail = substr $tail, length($1); $name = $p; push @rhs, $name; } elsif ($tail =~ m%^\s*!!%o) { # Exhortation to see the syntax rules - usually. my $str = "/* $tail */"; push @rhs, $str; last; } else { # Unknown! print "/* UNK: $tail */\n"; print STDERR "UNK:$.: $tail\n"; last; } } return(@rhs); } # Format a Yacc rule given LHS and RHS array sub record_rule { my($lhs, $comment, @rule) = @_; my($production) = ""; print "==>> record_rule ($lhs : @rule)\n" if debug; $production .= "/*\n" if $comment; $production .= "$lhs\n\t:\t"; my $pad = ""; my $br_count = 0; for (my $i = 0; $i <= $#rule; $i++) { my $item = $rule[$i]; print "==== item $item\n" if debug; if ($item eq "|" && $br_count == 0) { $production .= "\n\t|\t"; $pad = ""; } else { $production .= "$pad$item"; $pad = " "; $br_count++ if ($item eq '[' or $item eq '{'); $br_count-- if ($item eq ']' or $item eq '}'); } } $production .= "\n\t;\n"; $production .= "*/\n" if $comment; $production .= "\n"; print "$production" if debug; push @grammar, $production; print "<<== record_rule\n" if debug; } sub print_iterator { my($lhs,$rhs) = @_; my($production) = ""; print "==>> print_iterator ($lhs $rhs)\n" if debug; $production .= "$lhs\n\t:\t$rhs\n\t|\t$lhs $rhs\n\t;\n\n"; print "<<== print_iterator\n" if debug; push @grammar, $production; } # Process an optional item enclosed in square brackets sub find_balanced_bracket { my($lhs,@rhs) = @_; my(@rule) = ( "/* Nothing */", "|"); print "==>> find_balanced_bracket ($lhs : @rhs)\n" if debug; while (my $name = shift @rhs) { print " name = $name\n" if debug; if ($name eq ']') { # Found closing bracket # Terminate search last; } elsif ($name eq '[') { # Found nested optional clause my $tag = new_non_terminal('opt_'); @rhs = find_balanced_bracket($tag, @rhs); push @rule, $tag; } elsif ($name eq '{') { # Found start of sequence my $tag = new_non_terminal('seq_'); @rhs = find_balanced_brace($tag, @rhs); push @rule, $tag; } elsif ($name eq '}') { # Found unbalanced close brace. # Error! } elsif ($name eq '...') { # Found iteration. my $tag = new_non_terminal('lst_'); print "==== find_balanced_bracket: iterator (@rule)\n" if debug; my($old) = pop @rule; push @rule, $tag; print "==== find_balanced_bracket: iterator ($tag/$old - @rule)\n" if debug; print_iterator($tag, $old); } else { $name =~ s/^#//; push @rule, $name; $used{$name} = 1; } } record_rule($lhs, 0, @rule); print "<<== find_balanced_bracket: @rhs)\n" if debug; return(@rhs); } # Process an sequence item enclosed in curly braces sub find_balanced_brace { my($lhs,@rhs) = @_; my(@rule); print "==>> find_balanced_brace ($lhs : @rhs)\n" if debug; while (my $name = shift @rhs) { print " name = $name\n" if debug; if ($name eq '}') { # Found closing brace # Terminate search last; } elsif ($name eq '[') { # Found nested optional clause my $tag = new_non_terminal('opt_'); @rhs = find_balanced_bracket($tag, @rhs); push @rule, $tag; } elsif ($name eq '{') { # Found start of sequence my $tag = new_non_terminal('seq_'); @rhs = find_balanced_brace($tag, @rhs); push @rule, $tag; } elsif ($name eq ']') { # Found unbalanced close brace. # Error! } elsif ($name eq '...') { # Found iteration. my $tag = new_non_terminal('lst_'); print "==== find_balanced_brace: iterator (@rule)\n" if debug; my($old) = pop @rule; push @rule, $tag; print "==== find_balanced_brace: iterator ($tag/$old - @rule)\n" if debug; print_iterator($tag, $old); } else { $name =~ s/^#//; push @rule, $name; $used{$name} = 1; } } record_rule($lhs, 0, @rule); print "<<== find_balanced_brace: @rhs)\n" if debug; return(@rhs); } # Note that the [ and { parts are nice and easy because they are # balanced operators. The iteration operator ... is much harder to # process because it is a trailing modifier. When processing the list # of symbols, you need to establish whether there is a trailing iterator # after the current symbol, and modify the behaviour appropriately. sub process_rhs { my($lhs, $tail) = @_; my(@rhs) = scan_rhs($tail); print "==>> process_rhs ($lhs : @rhs)\n" if debug; # List parsed rule in output only if debugging. record_rule($lhs, 1, @rhs) if debug; my(@rule); while (my $name = shift @rhs) { print "name = $name\n" if debug; if ($name eq '[') { my $tag = new_non_terminal('opt_'); @rhs = find_balanced_bracket($tag, @rhs); push @rule, $tag; } elsif ($name eq ']') { # Found a close bracket for something unbalanced. # Error! } elsif ($name eq '{') { # Start of mandatory sequence of items, possibly containing alternatives. my $tag = new_non_terminal('seq_'); @rhs = find_balanced_brace($tag, @rhs); push @rule, $tag; } elsif ($name eq '}') { # Found a close brace for something unbalanced. # Error! } elsif ($name eq '|') { # End of one alternative and start of a new one. print "==== process_rhs: alternative $name\n" if debug; push @rule, $name; } elsif ($name eq '...') { # Found iteration. my $tag = new_non_terminal('lst_'); my($old) = pop @rule; push @rule, $tag; print "==== process_rhs: iterator\n" if debug; print_iterator($tag, $old); } elsif ($name =~ m/^#/) { # Keyword token print "==== process_rhs: token $name\n" if debug; $name =~ s/^#//; push @rule, $name; } else { # Non-terminal (or comment) print "==== process_rhs: non-terminal $name\n" if debug; push @rule, $name; } } print "==== process_rhs: @rule\n" if debug; record_rule($lhs, 0, @rule); print "<<== process_rhs\n" if debug; } sub count_unmatched_keys { my($ref1, $ref2) = @_; my(%keys) = %$ref1; my(%match) = %$ref2; my($count) = 0; foreach my $key (keys %keys) { $count++ unless defined $match{$key}; } return $count; } # ------------------------------------------------------------ open INPUT, "cat @ARGV |" or die "$!"; $_ = ; exit 0 unless defined($_); chomp; $heading = "%{\n/*\n** $_\n*/\n%}\n\n" unless m/^\s*$/; # Commentary appears in column 1. # Continuations of rules have a blank in column 1. # Blank lines, dash lines and equals lines separate rules (are not embedded within them).. while () { chomp; print "DBG:$.: $_\n" if debug; next if /^===*$/o; next if /^\s*$/o; # Blank lines next if /^---*$/o; # Horizontal lines if (/^--/o) { # Various HTML pseudo-directives if (m%^--/?\w+\b%) { print "/* $' */\n" if $'; } elsif (/^--%start (\w+)/) { $start = $1; print "/* Start symbol - $start */\n"; } elsif (/^--##/) { print "/* $_ */\n"; } else { print "/* Unrecognized 2: $_ */\n"; } } elsif (/^@.#..Id:/) { # Convert what(1) string identifier into version information s%^@.#..Id: %%; s% \$$%%; s%,v % %; s%\w+ Exp( \w+)?$%%; my @words = split; print "/*\n"; print "** Derived from file $words[0] version $words[1] dated $words[2] $words[3]\n"; print "*/\n"; } elsif (/ ::=/) { # Definition line my $def = $_; $def =~ s%<([-:/()\w\s]+)>.*%$1%o; $def = map_non_terminal($def); $rules{$def} = 1; $nonterminals{$def} = 1; my $tail = $_; $tail =~ s%.*::=\s*%%; # Remove LHS of statement while () { chomp; last unless /^\s/; $tail .= $_; } process_rhs($def, $tail); } else { # Anything unrecognized passed through as a comment! print "/* $_ */\n"; } } close INPUT; print "==== End of input phase ====\n" if debug; print $heading if $heading; # List of tokens foreach my $token (sort keys %tokens) { print "\%token $token\n"; } print "\n"; # Undefined non-terminals might need to be treated as tokens if (count_unmatched_keys(\%nonterminals, \%rules) > 0) { print "/* The following non-terminals were not defined */\n"; foreach my $nt (sort keys %nonterminals) { print "%token $nt\n" unless defined $rules{$nt}; } print "/* End of undefined non-terminals */\n\n"; } # List the rules that are defined in the original grammar. # Do not list the rules defined by this conversion process. print "/*\n"; foreach my $nt (sort keys %nonterminals) { print "\%rule $nt\n"; } print "*/\n\n"; if (defined $start) { print "%start $start\n\n"; print "%%\n\n"; } else { # No start symbol defined - let's see if we can work out what to use. # If there's more than one unused non-terminal, then treat them # all as simple alternatives to a list of statements. my $count = count_unmatched_keys(\%nonterminals, \%used); if ($count > 1) { my $prog = "bnf_program"; my $stmt = "bnf_statement"; print "%start $prog\n\n"; print "%%\n\n"; print "$prog\n\t:\t$stmt\n\t|\t$prog $stmt\n\t;\n\n"; print "$stmt\n"; my $pad = "\t:\t"; foreach my $nt (sort keys %nonterminals) { unless (defined $used{$nt}) { print "$pad$nt\n"; $pad = "\t|\t"; } } print "\t;\n\n"; } elsif ($count == 1) { foreach my $nt (sort keys %nonterminals) { print "%start $nt" unless defined $used{$nt}; } print "%%\n\n"; } else { # No single start symbol - loop? # Error! print STDERR "$0: no start symbol recognized!\n"; print "%%\n\n"; } } # Output the complete grammar while (my $line = shift @grammar) { print $line; } print "\n%%\n\n"; __END__ =pod Given a rule: abc: def ghi jkl The Yacc output is: abc : def ghi jkl ; Given a rule: abc: def [ ghi ] jkl The Yacc output is: abc : def opt_nt_0001 jkl ; opt_nt_0001 : /* Nothing */ | ghi ; Given a rule: abc: def { ghi } jkl The Yacc output is: abc : def seq_nt_0002 jkl ; seq_nt_0002 : ghi ; Note that such rules are seldom used in isolation; either the contents of the '{' to '}' contains alternatives, or the construct as a whole is followed by a repetition. Given a rule: abc: def | ghi The Yacc output is: abc : def | ghi ; Given a rule: abc: def ghi... jkl The Yacc output is: abc : def lst_nt_0003 jkl ; lst_nt_0003 : ghi | lst_nt_0003 ghi ; These rules can be, and often are, combined. The following examples come from the SQL-99 grammar which is the target of this effort. The target of this program is to produce Yacc rules equivalent to those which follow each fragment. Note that keywords (equivalently, terminals) are in upper case only; mixed case or lower case symbols are non-terminals. ::= [ ] [ ] [ ... ] ... SQL_client_module_definition : module_name_clause language_clause module_authorization_clause opt_nt_0001 opt_nt_0002 opt_nt_0003 lst_nt_0004 ; opt_nt_0001 : /* Nothing */ | module_path_specification ; opt_nt_0002 : /* Nothing */ | module_transform_group_specification ; opt_nt_0003 : /* Nothing */ | lst_nt_0005 ; lst_nt_0004 : module_contents | lst_nt_0004 module_contents ; lst_nt_0005 : temporary_table_declaration | lst_nt_0005 temporary_table_declaration ; The next example is interesting - it is fairly typical of the grammar, but is not minimal. The rule could be written ' ::= [ ... ]' without altering the meaning. It is not clear whether this program should apply this transformation automatically. ::= [ { }... ] identifier_body : identifier_start opt_nt_0006 ; opt_nt_0006 : /* Nothing */ | lst_nt_0007 ; lst_nt_0007 : seq_nt_0008 | lst_nt_0007 seq_nt_0008 ; seq_nt_0008 : identifier_part ; /* Optimized alternative to lst_nt_0007 */ lst_nt_0007 : identifier_part | lst_nt_0007 identifier_part ; ::= [ { | }... ] sql_language_identifier : sql_language_identifier_start opt_nt_0009 ; opt_nt_0009 : /* Nothing */ | lst_nt_0010 ; lst_nt_0010 : seq_nt_0011 | lst_nt_0010 seq_nt_0011 ; seq_nt_0011 : underscore | sql_language_identifier_part ; The next rule is the first example with keywords. ::= SCHEMA | AUTHORIZATION | SCHEMA AUTHORIZATION module_authorization_clause : SCHEMA schema_name | AUTHORIZATION module_authorization_identifier | SCHEMA schema_name AUTHORIZATION module_authorization_identifier ; ::= TRANSFORM GROUP { | } transform_group_specification : TRANSFORM GROUP seq_nt_0012 ; seq_nt_0012 : single_group_specification | multiple_group_specification ; ::= [ { }... ] multiple_group_specification : group_specification opt_nt_0013 ; opt_nt_0013 : /* Nothing */ | lst_nt_0014 ; lst_nt_0014 : seq_nt_0015 | lst_nt_0014 seq_nt_0015 ; seq_nt_0015 : comma group_specification ; Except for the presence of a token () after the optional list, the next example is equivalent to the previous one. It does show, however, that there is an element of lookahead required to tell whether an optional item contains a list or a sequence or a simple list of terminals and non-terminals. ::=
[ {
}... ] table_element_list : left_paren table_element opt_nt_0016 right_paren ; opt_nt_0016 : /* Nothing */ | lst_nt_0017 ; lst_nt_0017 : seq_nt_0018 | lst_nt_0017 seq_nt_0018 ; seq_nt_0018 : comma table_element ; The next example is interesting because the sequence item contains alternatives with no optionality or iteration. It suggests that the term 'sequence' is not necessarily the 'mot juste'. ::= { | } [ ] [ ] [ ... ] [ ] column_definition : column_name seq_nt_0019 opt_nt_0020 opt_nt_0021 opt_nt_0022 opt_nt_0023 ; seq_nt_0019 : data_type | domain_name ; opt_nt_0020 : /* Nothing */ | reference_scope_check ; opt_nt_0021 : /* Nothing */ | default_clause ; opt_nt_0022 : /* Nothing */ | lst_nt_0024 ; opt_nt_0023 : /* Nothing */ | collate_clause ; lst_nt_0024 : column_constraint_definition | lst_nt_0024 column_constraint_definition ; [ {