From 4b9958304de3325b34ba5a68cd5b34fd4aa81efa Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 9 Oct 2015 11:25:45 +0200 Subject: [PATCH 1/9] tools: add assembleSnippets.pl and test --- src/tools/assembleSnippets.pl | 131 ++++++++++++++++++++++++++++++++++ src/tools/test/Makefile | 1 + src/tools/test/Snippets.plt | 93 ++++++++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 src/tools/assembleSnippets.pl create mode 100644 src/tools/test/Snippets.plt diff --git a/src/tools/assembleSnippets.pl b/src/tools/assembleSnippets.pl new file mode 100644 index 000000000..30f8d2db0 --- /dev/null +++ b/src/tools/assembleSnippets.pl @@ -0,0 +1,131 @@ +#!/usr/bin/env perl +#************************************************************************* +# Copyright (c) 2015 ITER Organization. +# EPICS BASE is distributed subject to a Software License Agreement found +# in file LICENSE that is included with this distribution. +#************************************************************************* + +# $Id$ + +use strict; +use warnings; + +use Getopt::Std; +use File::Basename; +use Data::Dumper; + +our ($opt_o, $opt_d, $opt_m, $opt_i); + +$Getopt::Std::OUTPUT_HELP_VERSION = 1; +&HELP_MESSAGE if !getopts('i:m:o:d') || @ARGV == 0; + +my $out; +my %snippets; +my $ipattern; + +my $datetime = localtime(); +(my $user, my @dummy) = getpwuid($<); +my %replacements = ( + _DATETIME_ => $datetime, + _USER_ => $user, +); + +if ($opt_o) { + open $out, '>', $opt_o or + die "Can't create $opt_o: $!\n"; + print STDERR "opened file $opt_o for output\n" if $opt_d; +} else { + open $out, '>&', STDOUT; + print STDERR "using STDOUT for output\n" if $opt_d; +} + +if ($opt_m) { + foreach my $r (split /,/, $opt_m) { + (my $k, my $v) = split /=/, $r; + $replacements{$k} = $v; + } +} + +if ($opt_i) { + $ipattern = qr($opt_i); +} + +# %snippets is a hash {rank} +# of hashes {name-after-rank} +# of arrays[] [files...] +# of arrays[2] [filename, command] +print STDERR "reading input files\n" if $opt_d;; +foreach (@ARGV) { + my $name = basename($_); + if ($opt_i and not $name =~ /$ipattern/) { + print STDERR " snippet $_ does not match input pattern $opt_i - ignoring\n" if $opt_d; + next; + } + if ($name =~ /\A([ARD]?)([0-9]+)(.*[^~])\z/) { + print STDERR " considering snippet $_\n" if $opt_d; + if (exists $snippets{$2}) { + my %rank = %{$snippets{$2}}; + my @files = @{ $rank{(keys %rank)[0]} }; + my $existcmd = $files[0]->[1]; + if ($1 eq "D" and $existcmd ne "D") { + print STDERR " ignoring 'D' default for existing rank $2\n" if $opt_d; + next; + } elsif ($1 eq "R") { + print STDERR " 'R' command - deleting existing rank $2 snippets\n" if $opt_d; + $snippets{$2} = {}; + } elsif ($existcmd eq "D") { + print STDERR " deleting existing rank $2 default snippet\n" if $opt_d; + $snippets{$2} = {}; + } + } + if ($opt_d) { + print STDERR " adding snippet "; + print STDERR "marked as default " if $1 eq "D"; + print STDERR "to rank $2\n"; + } + $snippets{$2}{$3} = () if (not exists $snippets{$2}{$3}); + push @{$snippets{$2}{$3}}, [ $_, $1 ]; + } +} + +if ($opt_d) { + print STDERR "finished reading input files\n"; + print STDERR "dumping the final snippet structure\n"; + print STDERR Dumper(\%snippets); + print STDERR "dumping the macro replacements\n"; + print STDERR Dumper(\%replacements); + print STDERR "creating output\n"; +} + +foreach my $r (sort {$a<=>$b} keys %snippets) { + print STDERR " working on rank $r\n" if $opt_d; + foreach my $n (sort keys %{$snippets{$r}}) { + foreach my $s (@{$snippets{$r}{$n}}) { + my $in; + my $f = $s->[0]; + print STDERR " snippet $n from file $f\n" if $opt_d; + open $in, '<', $f or die "Can't open $f: $!\n"; + while (<$in>) { + chomp; + foreach my $k (keys %replacements) { + s/$k/$replacements{$k}/g; + } + print $out $_, "\n"; + } + close $in; + } + } +} + +print STDERR "finished creating output, closing\n" if $opt_d; +close $out; + +sub HELP_MESSAGE { + print STDERR "Usage: assembleSnippets.pl [options] snippets ...\n"; + print STDERR "Options:\n"; + print STDERR " -o file output file [STDOUT]\n"; + print STDERR " -d debug mode [no]\n"; + print STDERR " -m macros list of macro replacements as \"key=val,key=val\"\n"; + print STDERR " -i pattern pattern for input files to match\n"; + exit 2; +} diff --git a/src/tools/test/Makefile b/src/tools/test/Makefile index b0864e38a..2fed19509 100644 --- a/src/tools/test/Makefile +++ b/src/tools/test/Makefile @@ -18,6 +18,7 @@ TESTS += Menu TESTS += Recfield TESTS += Recordtype TESTS += Registrar +TESTS += Snippets TESTS += Variable TESTSCRIPTS_HOST += $(TESTS:%=%.t) diff --git a/src/tools/test/Snippets.plt b/src/tools/test/Snippets.plt new file mode 100644 index 000000000..792ba3a17 --- /dev/null +++ b/src/tools/test/Snippets.plt @@ -0,0 +1,93 @@ +#!/usr/bin/env perl + +use File::Path; + +use Test::More tests => 23; + +(my $user, my @rest) = getpwuid($<); + +mkdir "a$$"; +mkdir "b$$"; + +sub mksnip { + my ($dir, $file, $line) = @_; + open(my $fh, '>', "$dir$$/$file") or die "can't open $dir$$/$file : $!"; + print $fh $line; + close $fh; +} + +sub assemble { + my @cmd = ( 'perl', '../../assembleSnippets.pl', '-o', "out$$" ); + push @cmd, @_; + system(@cmd); + open(my $fh, '<', "out$$") or die "can't open out$$ : $!"; + chomp(my @result = <$fh>); + close $fh; + return join (' ', @result); +} + +# Adding two snippets of same rank, sorting alphabetically +mksnip('a', '10_a', '10'); +mksnip('b', '10_c', '12'); +is assemble("a$$/10_a", "b$$/10_c"), '10 12', "adding same rank; ordered"; +is assemble("b$$/10_c", "a$$/10_a"), '10 12', "adding same rank; reverse order"; + +# Same, with 'A' cmd +mksnip('a', 'A10_a', '10'); +mksnip('b', 'A10_c', '12'); +is assemble("a$$/10_a", "b$$/A10_c"), '10 12', "'A' add same rank; ordered"; +is assemble("b$$/10_c", "a$$/A10_a"), '10 12', "'A' add same rank; reverse order"; + +# Same name does not create issues +mksnip('b', '10_a', '10x'); +is assemble("a$$/10_a", "b$$/10_a"), '10 10x', "adding same name twice; order a-b"; +is assemble("b$$/10_a", "a$$/10_a"), '10x 10', "adding same name twice; order b-a"; + +# Backup files (trailing ~) and hidden files (leading '.') get ignored +mksnip('b', '10_c~', '12~'); +mksnip('b', '.10_c', '.12'); +is assemble("b$$/10_c", "b$$/10_c~"), '12', "backup file (trailing ~) gets ignored"; +is assemble("b$$/10_c", "b$$/.10_c"), '12', "hidden file (leading .) gets ignored"; + +# Non-numeric filenames get ignored +mksnip('a', 'foo10_a', 'foo10'); +is assemble("b$$/10_c", "a$$/foo10_a"), '12', "file starting with [^ADR0-9] gets ignored"; + +# 'R' command replaces existing snippets of same rank +mksnip('a', 'R10_b', '11'); +is assemble("a$$/10_a", "b$$/10_c", "a$$/R10_b"), '11', "'R' cmd; replace all"; +is assemble("a$$/10_a", "a$$/R10_b", "b$$/10_c"), '11 12', "'R' cmd; replace one (ordered)"; +is assemble("b$$/10_c", "a$$/R10_b", "a$$/10_a"), '10 11', "'R' cmd; replace one (reverse order)"; + +# 'D' command establishes default that gets overwritten or ignored +mksnip('a', 'D10_a', 'D10'); +mksnip('b', 'D10_c', 'D12'); +is assemble("a$$/D10_a", "b$$/10_c"), '12', "'D' default; replaced by regular"; +is assemble("a$$/D10_a", "b$$/D10_c"), 'D12', "'D' default; replaced by new default (ordered)"; +is assemble("b$$/D10_c", "a$$/D10_a"), 'D10', "'D' default; replaced by new default (reverse order)"; +is assemble("a$$/D10_a", "a$$/R10_b"), '11', "'D' default; replaced by 'R' cmd"; +is assemble("b$$/10_c", "a$$/D10_a"), '12', "'D' default; ignored when regular exists"; + +# Ranks are sorted numerically +mksnip('b', '09_a', '09'); +mksnip('a', '15_a', '15'); +mksnip('b', '2_a', '2'); +is assemble("a$$/10_a", "b$$/2_a", "a$$/15_a", "b$$/09_a"), '2 09 10 15', "ranks are sorted numerically"; + +# Builtin macros +mksnip('a', '30_a', '_USER_'); +is assemble("a$$/30_a"), "$user", "builtin macro _USER_"; + +# User macros +mksnip('b', '35_a', 'Line _M1_'); +mksnip('b', '35_b', 'Line _M1_ with _M2_'); +mksnip('b', '35_c', 'Line _M2_ with _M2_'); +is assemble("-m", "_M1_=REP1", "b$$/35_a"), "Line REP1", "single user macro; single occurrence"; +is assemble("-m", "_M1_=REP1,_M2_=REP2", "b$$/35_b"), "Line REP1 with REP2", "multiple user macros"; +is assemble("-m", "_M2_=REP2", "b$$/35_c"), "Line REP2 with REP2", "single user macro; multiple occurrences"; + +# Input pattern +mksnip('a', '10_a.cmd', '10cmd'); +is assemble("-i", "\.cmd", "a$$/10_a", "b$$/10_c", "a$$/R10_b", "a$$/10_a.cmd"), '10cmd', "'input pattern"; + +rmtree([ "a$$", "b$$", "out$$" ]); From c2d2f671bb27ba469e04610717bbb055bf5d6bd6 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 9 Oct 2015 14:15:26 +0200 Subject: [PATCH 2/9] tools: add -M option to assembleSnippets.pl (creates dependency file) --- src/tools/assembleSnippets.pl | 20 +++++++++++++++++--- src/tools/test/Snippets.plt | 15 ++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/tools/assembleSnippets.pl b/src/tools/assembleSnippets.pl index 30f8d2db0..4a62e67ec 100644 --- a/src/tools/assembleSnippets.pl +++ b/src/tools/assembleSnippets.pl @@ -14,12 +14,13 @@ use Getopt::Std; use File::Basename; use Data::Dumper; -our ($opt_o, $opt_d, $opt_m, $opt_i); +our ($opt_o, $opt_d, $opt_m, $opt_i, $opt_M); $Getopt::Std::OUTPUT_HELP_VERSION = 1; -&HELP_MESSAGE if !getopts('i:m:o:d') || @ARGV == 0; +&HELP_MESSAGE if !getopts('M:i:m:o:d') || @ARGV == 0; my $out; +my $dep; my %snippets; my $ipattern; @@ -46,6 +47,13 @@ if ($opt_m) { } } +if ($opt_M) { + open $dep, '>', $opt_M or + die "Can't create $opt_M: $!\n"; + print STDERR "opened dependency file $opt_M for output\n" if $opt_d; + print $dep basename($opt_o), ":"; +} + if ($opt_i) { $ipattern = qr($opt_i); } @@ -54,7 +62,7 @@ if ($opt_i) { # of hashes {name-after-rank} # of arrays[] [files...] # of arrays[2] [filename, command] -print STDERR "reading input files\n" if $opt_d;; +print STDERR "reading input files\n" if $opt_d; foreach (@ARGV) { my $name = basename($_); if ($opt_i and not $name =~ /$ipattern/) { @@ -105,6 +113,7 @@ foreach my $r (sort {$a<=>$b} keys %snippets) { my $f = $s->[0]; print STDERR " snippet $n from file $f\n" if $opt_d; open $in, '<', $f or die "Can't open $f: $!\n"; + print $dep " \\\n $f" if $opt_M; while (<$in>) { chomp; foreach my $k (keys %replacements) { @@ -118,6 +127,10 @@ foreach my $r (sort {$a<=>$b} keys %snippets) { } print STDERR "finished creating output, closing\n" if $opt_d; +if ($opt_M) { + print $dep "\n"; + close $dep; +} close $out; sub HELP_MESSAGE { @@ -127,5 +140,6 @@ sub HELP_MESSAGE { print STDERR " -d debug mode [no]\n"; print STDERR " -m macros list of macro replacements as \"key=val,key=val\"\n"; print STDERR " -i pattern pattern for input files to match\n"; + print STDERR " -M file write file with dependency rule suitable for make\n"; exit 2; } diff --git a/src/tools/test/Snippets.plt b/src/tools/test/Snippets.plt index 792ba3a17..3f30f6593 100644 --- a/src/tools/test/Snippets.plt +++ b/src/tools/test/Snippets.plt @@ -2,7 +2,7 @@ use File::Path; -use Test::More tests => 23; +use Test::More tests => 26; (my $user, my @rest) = getpwuid($<); @@ -88,6 +88,15 @@ is assemble("-m", "_M2_=REP2", "b$$/35_c"), "Line REP2 with REP2", "single user # Input pattern mksnip('a', '10_a.cmd', '10cmd'); -is assemble("-i", "\.cmd", "a$$/10_a", "b$$/10_c", "a$$/R10_b", "a$$/10_a.cmd"), '10cmd', "'input pattern"; +is assemble("-i", "\.cmd", "a$$/10_a", "b$$/10_c", "a$$/R10_b", "a$$/10_a.cmd"), '10cmd', "input pattern"; -rmtree([ "a$$", "b$$", "out$$" ]); +# Dependency file generation +assemble("-M", "./dep$$", "a$$/10_a", "b$$/10_c"); +open(my $fh, '<', "dep$$") or die "can't open dep$$ : $!"; +chomp(my @result = <$fh>); +close $fh; +is "$result[0]", "out$$: \\", "dependency file (line 1)"; +is "$result[1]", " a$$/10_a \\", "dependency file (line 2)"; +is "$result[2]", " b$$/10_c", "dependency file (line 3)"; + +rmtree([ "a$$", "b$$", "out$$", "dep$$" ]); From 96ee2cd00c6b4c4a5d62670b0093d92e43961597 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 9 Oct 2015 15:28:40 +0200 Subject: [PATCH 3/9] tools: install assembleSnippets.pl --- src/tools/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/Makefile b/src/tools/Makefile index 0e66e49c9..e15e93507 100644 --- a/src/tools/Makefile +++ b/src/tools/Makefile @@ -32,6 +32,7 @@ PERL_MODULES += DBD/Recordtype.pm PERL_MODULES += DBD/Registrar.pm PERL_MODULES += DBD/Variable.pm +PERL_SCRIPTS += assembleSnippets.pl PERL_SCRIPTS += convertRelease.pl PERL_SCRIPTS += cvsclean.pl PERL_SCRIPTS += dos2unix.pl From 830704021d3df2359fdb7af5e9fbc86d828bf280 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Fri, 9 Oct 2015 17:28:46 +0200 Subject: [PATCH 4/9] Add creating config files from snippets to build system. --- configure/RULES_BUILD | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/configure/RULES_BUILD b/configure/RULES_BUILD index 37e54372f..e9ee9da3c 100644 --- a/configure/RULES_BUILD +++ b/configure/RULES_BUILD @@ -315,6 +315,22 @@ $(MODNAME): %$(MODEXT): %$(EXE) @$(RM) $@ $(LINK.mod) +#--------------------------------------------------------------- +# Assembled files (from snippets) + +COMMON_ASSEMBLE_FROM_SNIPPETS_ += $(addprefix $(COMMON_DIR)/,$(COMMON_ASSEMBLE_FROM_SNIPPETS)) + +$(COMMON_ASSEMBLE_FROM_SNIPPETS_): $(COMMON_DIR)/%: + @echo "Assembling common file $@ from snippets" + @$(RM) $(notdir $@) + $(PERL) $(TOOLS)/assembleSnippets.pl -o $(notdir $@) $($(notdir $@)_SNIPPETS) + @$(MV) $(notdir $@) $@ + +$(ASSEMBLE_FROM_SNIPPETS): %: + @echo "Assembling file $@ from snippets" + @$(RM) $(notdir $@) + $(PERL) $(TOOLS)/assembleSnippets.pl -o $(notdir $@) $($(notdir $@)_SNIPPETS) + #--------------------------------------------------------------- # Automated testing From 5af066305890f661db838f1e9628f0503be7c920 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Sat, 10 Oct 2015 18:17:35 +0200 Subject: [PATCH 5/9] Back out 12706 (assembly integration in build system) --- configure/RULES_BUILD | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/configure/RULES_BUILD b/configure/RULES_BUILD index e9ee9da3c..37e54372f 100644 --- a/configure/RULES_BUILD +++ b/configure/RULES_BUILD @@ -315,22 +315,6 @@ $(MODNAME): %$(MODEXT): %$(EXE) @$(RM) $@ $(LINK.mod) -#--------------------------------------------------------------- -# Assembled files (from snippets) - -COMMON_ASSEMBLE_FROM_SNIPPETS_ += $(addprefix $(COMMON_DIR)/,$(COMMON_ASSEMBLE_FROM_SNIPPETS)) - -$(COMMON_ASSEMBLE_FROM_SNIPPETS_): $(COMMON_DIR)/%: - @echo "Assembling common file $@ from snippets" - @$(RM) $(notdir $@) - $(PERL) $(TOOLS)/assembleSnippets.pl -o $(notdir $@) $($(notdir $@)_SNIPPETS) - @$(MV) $(notdir $@) $@ - -$(ASSEMBLE_FROM_SNIPPETS): %: - @echo "Assembling file $@ from snippets" - @$(RM) $(notdir $@) - $(PERL) $(TOOLS)/assembleSnippets.pl -o $(notdir $@) $($(notdir $@)_SNIPPETS) - #--------------------------------------------------------------- # Automated testing From 88cb33a04d8eb77775814c146ffd2698bad9ba6f Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Sat, 10 Oct 2015 18:54:20 +0200 Subject: [PATCH 6/9] tools: fix in assembleSnippets script and test for Windows (getpwuid() not implemented) --- src/tools/assembleSnippets.pl | 290 +++++++++++++++++----------------- src/tools/test/Snippets.plt | 204 ++++++++++++------------ 2 files changed, 247 insertions(+), 247 deletions(-) diff --git a/src/tools/assembleSnippets.pl b/src/tools/assembleSnippets.pl index 4a62e67ec..520ec318b 100644 --- a/src/tools/assembleSnippets.pl +++ b/src/tools/assembleSnippets.pl @@ -1,145 +1,145 @@ -#!/usr/bin/env perl -#************************************************************************* -# Copyright (c) 2015 ITER Organization. -# EPICS BASE is distributed subject to a Software License Agreement found -# in file LICENSE that is included with this distribution. -#************************************************************************* - -# $Id$ - -use strict; -use warnings; - -use Getopt::Std; -use File::Basename; -use Data::Dumper; - -our ($opt_o, $opt_d, $opt_m, $opt_i, $opt_M); - -$Getopt::Std::OUTPUT_HELP_VERSION = 1; -&HELP_MESSAGE if !getopts('M:i:m:o:d') || @ARGV == 0; - -my $out; -my $dep; -my %snippets; -my $ipattern; - -my $datetime = localtime(); -(my $user, my @dummy) = getpwuid($<); -my %replacements = ( - _DATETIME_ => $datetime, - _USER_ => $user, -); - -if ($opt_o) { - open $out, '>', $opt_o or - die "Can't create $opt_o: $!\n"; - print STDERR "opened file $opt_o for output\n" if $opt_d; -} else { - open $out, '>&', STDOUT; - print STDERR "using STDOUT for output\n" if $opt_d; -} - -if ($opt_m) { - foreach my $r (split /,/, $opt_m) { - (my $k, my $v) = split /=/, $r; - $replacements{$k} = $v; - } -} - -if ($opt_M) { - open $dep, '>', $opt_M or - die "Can't create $opt_M: $!\n"; - print STDERR "opened dependency file $opt_M for output\n" if $opt_d; - print $dep basename($opt_o), ":"; -} - -if ($opt_i) { - $ipattern = qr($opt_i); -} - -# %snippets is a hash {rank} -# of hashes {name-after-rank} -# of arrays[] [files...] -# of arrays[2] [filename, command] -print STDERR "reading input files\n" if $opt_d; -foreach (@ARGV) { - my $name = basename($_); - if ($opt_i and not $name =~ /$ipattern/) { - print STDERR " snippet $_ does not match input pattern $opt_i - ignoring\n" if $opt_d; - next; - } - if ($name =~ /\A([ARD]?)([0-9]+)(.*[^~])\z/) { - print STDERR " considering snippet $_\n" if $opt_d; - if (exists $snippets{$2}) { - my %rank = %{$snippets{$2}}; - my @files = @{ $rank{(keys %rank)[0]} }; - my $existcmd = $files[0]->[1]; - if ($1 eq "D" and $existcmd ne "D") { - print STDERR " ignoring 'D' default for existing rank $2\n" if $opt_d; - next; - } elsif ($1 eq "R") { - print STDERR " 'R' command - deleting existing rank $2 snippets\n" if $opt_d; - $snippets{$2} = {}; - } elsif ($existcmd eq "D") { - print STDERR " deleting existing rank $2 default snippet\n" if $opt_d; - $snippets{$2} = {}; - } - } - if ($opt_d) { - print STDERR " adding snippet "; - print STDERR "marked as default " if $1 eq "D"; - print STDERR "to rank $2\n"; - } - $snippets{$2}{$3} = () if (not exists $snippets{$2}{$3}); - push @{$snippets{$2}{$3}}, [ $_, $1 ]; - } -} - -if ($opt_d) { - print STDERR "finished reading input files\n"; - print STDERR "dumping the final snippet structure\n"; - print STDERR Dumper(\%snippets); - print STDERR "dumping the macro replacements\n"; - print STDERR Dumper(\%replacements); - print STDERR "creating output\n"; -} - -foreach my $r (sort {$a<=>$b} keys %snippets) { - print STDERR " working on rank $r\n" if $opt_d; - foreach my $n (sort keys %{$snippets{$r}}) { - foreach my $s (@{$snippets{$r}{$n}}) { - my $in; - my $f = $s->[0]; - print STDERR " snippet $n from file $f\n" if $opt_d; - open $in, '<', $f or die "Can't open $f: $!\n"; - print $dep " \\\n $f" if $opt_M; - while (<$in>) { - chomp; - foreach my $k (keys %replacements) { - s/$k/$replacements{$k}/g; - } - print $out $_, "\n"; - } - close $in; - } - } -} - -print STDERR "finished creating output, closing\n" if $opt_d; -if ($opt_M) { - print $dep "\n"; - close $dep; -} -close $out; - -sub HELP_MESSAGE { - print STDERR "Usage: assembleSnippets.pl [options] snippets ...\n"; - print STDERR "Options:\n"; - print STDERR " -o file output file [STDOUT]\n"; - print STDERR " -d debug mode [no]\n"; - print STDERR " -m macros list of macro replacements as \"key=val,key=val\"\n"; - print STDERR " -i pattern pattern for input files to match\n"; - print STDERR " -M file write file with dependency rule suitable for make\n"; - exit 2; -} +#!/usr/bin/env perl +#************************************************************************* +# Copyright (c) 2015 ITER Organization. +# EPICS BASE is distributed subject to a Software License Agreement found +# in file LICENSE that is included with this distribution. +#************************************************************************* + +# $Id$ + +use strict; +use warnings; + +use Getopt::Std; +use File::Basename; +use Data::Dumper; + +our ($opt_o, $opt_d, $opt_m, $opt_i, $opt_M); + +$Getopt::Std::OUTPUT_HELP_VERSION = 1; +&HELP_MESSAGE if !getopts('M:i:m:o:d') || @ARGV == 0; + +my $out; +my $dep; +my %snippets; +my $ipattern; + +my $datetime = localtime(); +my $user = $ENV{LOGNAME} || $ENV{USER} || $ENV{USERNAME}; +my %replacements = ( + _DATETIME_ => $datetime, + _USER_ => $user, +); + +if ($opt_o) { + open $out, '>', $opt_o or + die "Can't create $opt_o: $!\n"; + print STDERR "opened file $opt_o for output\n" if $opt_d; +} else { + open $out, '>&', STDOUT; + print STDERR "using STDOUT for output\n" if $opt_d; +} + +if ($opt_m) { + foreach my $r (split /,/, $opt_m) { + (my $k, my $v) = split /=/, $r; + $replacements{$k} = $v; + } +} + +if ($opt_M) { + open $dep, '>', $opt_M or + die "Can't create $opt_M: $!\n"; + print STDERR "opened dependency file $opt_M for output\n" if $opt_d; + print $dep basename($opt_o), ":"; +} + +if ($opt_i) { + $ipattern = qr($opt_i); +} + +# %snippets is a hash {rank} +# of hashes {name-after-rank} +# of arrays[] [files...] +# of arrays[2] [filename, command] +print STDERR "reading input files\n" if $opt_d; +foreach (@ARGV) { + my $name = basename($_); + if ($opt_i and not $name =~ /$ipattern/) { + print STDERR " snippet $_ does not match input pattern $opt_i - ignoring\n" if $opt_d; + next; + } + if ($name =~ /\A([ARD]?)([0-9]+)(.*[^~])\z/) { + print STDERR " considering snippet $_\n" if $opt_d; + if (exists $snippets{$2}) { + my %rank = %{$snippets{$2}}; + my @files = @{ $rank{(keys %rank)[0]} }; + my $existcmd = $files[0]->[1]; + if ($1 eq "D" and $existcmd ne "D") { + print STDERR " ignoring 'D' default for existing rank $2\n" if $opt_d; + next; + } elsif ($1 eq "R") { + print STDERR " 'R' command - deleting existing rank $2 snippets\n" if $opt_d; + $snippets{$2} = {}; + } elsif ($existcmd eq "D") { + print STDERR " deleting existing rank $2 default snippet\n" if $opt_d; + $snippets{$2} = {}; + } + } + if ($opt_d) { + print STDERR " adding snippet "; + print STDERR "marked as default " if $1 eq "D"; + print STDERR "to rank $2\n"; + } + $snippets{$2}{$3} = () if (not exists $snippets{$2}{$3}); + push @{$snippets{$2}{$3}}, [ $_, $1 ]; + } +} + +if ($opt_d) { + print STDERR "finished reading input files\n"; + print STDERR "dumping the final snippet structure\n"; + print STDERR Dumper(\%snippets); + print STDERR "dumping the macro replacements\n"; + print STDERR Dumper(\%replacements); + print STDERR "creating output\n"; +} + +foreach my $r (sort {$a<=>$b} keys %snippets) { + print STDERR " working on rank $r\n" if $opt_d; + foreach my $n (sort keys %{$snippets{$r}}) { + foreach my $s (@{$snippets{$r}{$n}}) { + my $in; + my $f = $s->[0]; + print STDERR " snippet $n from file $f\n" if $opt_d; + open $in, '<', $f or die "Can't open $f: $!\n"; + print $dep " \\\n $f" if $opt_M; + while (<$in>) { + chomp; + foreach my $k (keys %replacements) { + s/$k/$replacements{$k}/g; + } + print $out $_, "\n"; + } + close $in; + } + } +} + +print STDERR "finished creating output, closing\n" if $opt_d; +if ($opt_M) { + print $dep "\n"; + close $dep; +} +close $out; + +sub HELP_MESSAGE { + print STDERR "Usage: assembleSnippets.pl [options] snippets ...\n"; + print STDERR "Options:\n"; + print STDERR " -o file output file [STDOUT]\n"; + print STDERR " -d debug mode [no]\n"; + print STDERR " -m macros list of macro replacements as \"key=val,key=val\"\n"; + print STDERR " -i pattern pattern for input files to match\n"; + print STDERR " -M file write file with dependency rule suitable for make\n"; + exit 2; +} diff --git a/src/tools/test/Snippets.plt b/src/tools/test/Snippets.plt index 3f30f6593..eaf1a263a 100644 --- a/src/tools/test/Snippets.plt +++ b/src/tools/test/Snippets.plt @@ -1,102 +1,102 @@ -#!/usr/bin/env perl - -use File::Path; - -use Test::More tests => 26; - -(my $user, my @rest) = getpwuid($<); - -mkdir "a$$"; -mkdir "b$$"; - -sub mksnip { - my ($dir, $file, $line) = @_; - open(my $fh, '>', "$dir$$/$file") or die "can't open $dir$$/$file : $!"; - print $fh $line; - close $fh; -} - -sub assemble { - my @cmd = ( 'perl', '../../assembleSnippets.pl', '-o', "out$$" ); - push @cmd, @_; - system(@cmd); - open(my $fh, '<', "out$$") or die "can't open out$$ : $!"; - chomp(my @result = <$fh>); - close $fh; - return join (' ', @result); -} - -# Adding two snippets of same rank, sorting alphabetically -mksnip('a', '10_a', '10'); -mksnip('b', '10_c', '12'); -is assemble("a$$/10_a", "b$$/10_c"), '10 12', "adding same rank; ordered"; -is assemble("b$$/10_c", "a$$/10_a"), '10 12', "adding same rank; reverse order"; - -# Same, with 'A' cmd -mksnip('a', 'A10_a', '10'); -mksnip('b', 'A10_c', '12'); -is assemble("a$$/10_a", "b$$/A10_c"), '10 12', "'A' add same rank; ordered"; -is assemble("b$$/10_c", "a$$/A10_a"), '10 12', "'A' add same rank; reverse order"; - -# Same name does not create issues -mksnip('b', '10_a', '10x'); -is assemble("a$$/10_a", "b$$/10_a"), '10 10x', "adding same name twice; order a-b"; -is assemble("b$$/10_a", "a$$/10_a"), '10x 10', "adding same name twice; order b-a"; - -# Backup files (trailing ~) and hidden files (leading '.') get ignored -mksnip('b', '10_c~', '12~'); -mksnip('b', '.10_c', '.12'); -is assemble("b$$/10_c", "b$$/10_c~"), '12', "backup file (trailing ~) gets ignored"; -is assemble("b$$/10_c", "b$$/.10_c"), '12', "hidden file (leading .) gets ignored"; - -# Non-numeric filenames get ignored -mksnip('a', 'foo10_a', 'foo10'); -is assemble("b$$/10_c", "a$$/foo10_a"), '12', "file starting with [^ADR0-9] gets ignored"; - -# 'R' command replaces existing snippets of same rank -mksnip('a', 'R10_b', '11'); -is assemble("a$$/10_a", "b$$/10_c", "a$$/R10_b"), '11', "'R' cmd; replace all"; -is assemble("a$$/10_a", "a$$/R10_b", "b$$/10_c"), '11 12', "'R' cmd; replace one (ordered)"; -is assemble("b$$/10_c", "a$$/R10_b", "a$$/10_a"), '10 11', "'R' cmd; replace one (reverse order)"; - -# 'D' command establishes default that gets overwritten or ignored -mksnip('a', 'D10_a', 'D10'); -mksnip('b', 'D10_c', 'D12'); -is assemble("a$$/D10_a", "b$$/10_c"), '12', "'D' default; replaced by regular"; -is assemble("a$$/D10_a", "b$$/D10_c"), 'D12', "'D' default; replaced by new default (ordered)"; -is assemble("b$$/D10_c", "a$$/D10_a"), 'D10', "'D' default; replaced by new default (reverse order)"; -is assemble("a$$/D10_a", "a$$/R10_b"), '11', "'D' default; replaced by 'R' cmd"; -is assemble("b$$/10_c", "a$$/D10_a"), '12', "'D' default; ignored when regular exists"; - -# Ranks are sorted numerically -mksnip('b', '09_a', '09'); -mksnip('a', '15_a', '15'); -mksnip('b', '2_a', '2'); -is assemble("a$$/10_a", "b$$/2_a", "a$$/15_a", "b$$/09_a"), '2 09 10 15', "ranks are sorted numerically"; - -# Builtin macros -mksnip('a', '30_a', '_USER_'); -is assemble("a$$/30_a"), "$user", "builtin macro _USER_"; - -# User macros -mksnip('b', '35_a', 'Line _M1_'); -mksnip('b', '35_b', 'Line _M1_ with _M2_'); -mksnip('b', '35_c', 'Line _M2_ with _M2_'); -is assemble("-m", "_M1_=REP1", "b$$/35_a"), "Line REP1", "single user macro; single occurrence"; -is assemble("-m", "_M1_=REP1,_M2_=REP2", "b$$/35_b"), "Line REP1 with REP2", "multiple user macros"; -is assemble("-m", "_M2_=REP2", "b$$/35_c"), "Line REP2 with REP2", "single user macro; multiple occurrences"; - -# Input pattern -mksnip('a', '10_a.cmd', '10cmd'); -is assemble("-i", "\.cmd", "a$$/10_a", "b$$/10_c", "a$$/R10_b", "a$$/10_a.cmd"), '10cmd', "input pattern"; - -# Dependency file generation -assemble("-M", "./dep$$", "a$$/10_a", "b$$/10_c"); -open(my $fh, '<', "dep$$") or die "can't open dep$$ : $!"; -chomp(my @result = <$fh>); -close $fh; -is "$result[0]", "out$$: \\", "dependency file (line 1)"; -is "$result[1]", " a$$/10_a \\", "dependency file (line 2)"; -is "$result[2]", " b$$/10_c", "dependency file (line 3)"; - -rmtree([ "a$$", "b$$", "out$$", "dep$$" ]); +#!/usr/bin/env perl + +use File::Path; + +use Test::More tests => 26; + +my $user = $ENV{LOGNAME} || $ENV{USER} || $ENV{USERNAME}; + +mkdir "a$$"; +mkdir "b$$"; + +sub mksnip { + my ($dir, $file, $line) = @_; + open(my $fh, '>', "$dir$$/$file") or die "can't open $dir$$/$file : $!"; + print $fh $line; + close $fh; +} + +sub assemble { + my @cmd = ( 'perl', '../../assembleSnippets.pl', '-o', "out$$" ); + push @cmd, @_; + system(@cmd); + open(my $fh, '<', "out$$") or die "can't open out$$ : $!"; + chomp(my @result = <$fh>); + close $fh; + return join (' ', @result); +} + +# Adding two snippets of same rank, sorting alphabetically +mksnip('a', '10_a', '10'); +mksnip('b', '10_c', '12'); +is assemble("a$$/10_a", "b$$/10_c"), '10 12', "adding same rank; ordered"; +is assemble("b$$/10_c", "a$$/10_a"), '10 12', "adding same rank; reverse order"; + +# Same, with 'A' cmd +mksnip('a', 'A10_a', '10'); +mksnip('b', 'A10_c', '12'); +is assemble("a$$/10_a", "b$$/A10_c"), '10 12', "'A' add same rank; ordered"; +is assemble("b$$/10_c", "a$$/A10_a"), '10 12', "'A' add same rank; reverse order"; + +# Same name does not create issues +mksnip('b', '10_a', '10x'); +is assemble("a$$/10_a", "b$$/10_a"), '10 10x', "adding same name twice; order a-b"; +is assemble("b$$/10_a", "a$$/10_a"), '10x 10', "adding same name twice; order b-a"; + +# Backup files (trailing ~) and hidden files (leading '.') get ignored +mksnip('b', '10_c~', '12~'); +mksnip('b', '.10_c', '.12'); +is assemble("b$$/10_c", "b$$/10_c~"), '12', "backup file (trailing ~) gets ignored"; +is assemble("b$$/10_c", "b$$/.10_c"), '12', "hidden file (leading .) gets ignored"; + +# Non-numeric filenames get ignored +mksnip('a', 'foo10_a', 'foo10'); +is assemble("b$$/10_c", "a$$/foo10_a"), '12', "file starting with [^ADR0-9] gets ignored"; + +# 'R' command replaces existing snippets of same rank +mksnip('a', 'R10_b', '11'); +is assemble("a$$/10_a", "b$$/10_c", "a$$/R10_b"), '11', "'R' cmd; replace all"; +is assemble("a$$/10_a", "a$$/R10_b", "b$$/10_c"), '11 12', "'R' cmd; replace one (ordered)"; +is assemble("b$$/10_c", "a$$/R10_b", "a$$/10_a"), '10 11', "'R' cmd; replace one (reverse order)"; + +# 'D' command establishes default that gets overwritten or ignored +mksnip('a', 'D10_a', 'D10'); +mksnip('b', 'D10_c', 'D12'); +is assemble("a$$/D10_a", "b$$/10_c"), '12', "'D' default; replaced by regular"; +is assemble("a$$/D10_a", "b$$/D10_c"), 'D12', "'D' default; replaced by new default (ordered)"; +is assemble("b$$/D10_c", "a$$/D10_a"), 'D10', "'D' default; replaced by new default (reverse order)"; +is assemble("a$$/D10_a", "a$$/R10_b"), '11', "'D' default; replaced by 'R' cmd"; +is assemble("b$$/10_c", "a$$/D10_a"), '12', "'D' default; ignored when regular exists"; + +# Ranks are sorted numerically +mksnip('b', '09_a', '09'); +mksnip('a', '15_a', '15'); +mksnip('b', '2_a', '2'); +is assemble("a$$/10_a", "b$$/2_a", "a$$/15_a", "b$$/09_a"), '2 09 10 15', "ranks are sorted numerically"; + +# Builtin macros +mksnip('a', '30_a', '_USER_'); +is assemble("a$$/30_a"), "$user", "builtin macro _USER_"; + +# User macros +mksnip('b', '35_a', 'Line _M1_'); +mksnip('b', '35_b', 'Line _M1_ with _M2_'); +mksnip('b', '35_c', 'Line _M2_ with _M2_'); +is assemble("-m", "_M1_=REP1", "b$$/35_a"), "Line REP1", "single user macro; single occurrence"; +is assemble("-m", "_M1_=REP1,_M2_=REP2", "b$$/35_b"), "Line REP1 with REP2", "multiple user macros"; +is assemble("-m", "_M2_=REP2", "b$$/35_c"), "Line REP2 with REP2", "single user macro; multiple occurrences"; + +# Input pattern +mksnip('a', '10_a.cmd', '10cmd'); +is assemble("-i", "\.cmd", "a$$/10_a", "b$$/10_c", "a$$/R10_b", "a$$/10_a.cmd"), '10cmd', "input pattern"; + +# Dependency file generation +assemble("-M", "./dep$$", "a$$/10_a", "b$$/10_c"); +open(my $fh, '<', "dep$$") or die "can't open dep$$ : $!"; +chomp(my @result = <$fh>); +close $fh; +is "$result[0]", "out$$: \\", "dependency file (line 1)"; +is "$result[1]", " a$$/10_a \\", "dependency file (line 2)"; +is "$result[2]", " b$$/10_c", "dependency file (line 3)"; + +rmtree([ "a$$", "b$$", "out$$", "dep$$" ]); From 8596dc41f290907f9beca22ca5e57609cb2cf098 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Sat, 10 Oct 2015 18:55:17 +0200 Subject: [PATCH 7/9] configure: add assemblies to RULES_EXPAND --- configure/RULES_EXPAND | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/configure/RULES_EXPAND b/configure/RULES_EXPAND index ca087aadc..69636db19 100644 --- a/configure/RULES_EXPAND +++ b/configure/RULES_EXPAND @@ -2,6 +2,9 @@ vpath %@ $(USR_VPATH) $(ALL_SRC_DIRS) +#--------------------------------------------------------------- +# Variable expansion + # Default settings EXPAND_TOOL ?= $(PERL) $(TOOLS)/expandVars.pl @@ -24,3 +27,27 @@ expand_clean: @$(RM) $(EXPANDED) .PHONY : expand_clean + +#--------------------------------------------------------------- +# Assemblies (files assembled from snippets) + +ASSEMBLE_TOOL ?= $(PERL) $(TOOLS)/assembleSnippets.pl + +define COMMON_ASSEMBLY_template +$(COMMON_DIR)/$1: $($1_SNIPPETS) + @echo "Assembling file $$@ from snippets" + @$(RM) $1 + $(ASSEMBLE_TOOL) -o $1 $($1_SNIPPETS) + @$(MV) $1 $$@ +endef +$(foreach asy, $(COMMON_ASSEMBLIES), \ + $(eval $(call COMMON_ASSEMBLY_template,$(strip $(asy))))) + +define ASSEMBLY_template +$1: $($1_SNIPPETS) + @echo "Assembling file $$@ from snippets" + @$(RM) $$@ + $(ASSEMBLE_TOOL) -o $$@ $($1_SNIPPETS) +endef +$(foreach asy, $(ASSEMBLIES), \ + $(eval $(call ASSEMBLY_template,$(strip $(asy))))) From f8b0b2f5f0224d42123c10fb9fe5542f124a8d28 Mon Sep 17 00:00:00 2001 From: Ralph Lange Date: Sat, 10 Oct 2015 19:12:11 +0200 Subject: [PATCH 8/9] tools: add new builtin macros to assemblies --- src/tools/assembleSnippets.pl | 6 ++++++ src/tools/test/Snippets.plt | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/tools/assembleSnippets.pl b/src/tools/assembleSnippets.pl index 520ec318b..ae9ce1acc 100644 --- a/src/tools/assembleSnippets.pl +++ b/src/tools/assembleSnippets.pl @@ -11,6 +11,7 @@ use strict; use warnings; use Getopt::Std; +use Sys::Hostname; use File::Basename; use Data::Dumper; @@ -26,18 +27,22 @@ my $ipattern; my $datetime = localtime(); my $user = $ENV{LOGNAME} || $ENV{USER} || $ENV{USERNAME}; +my $host = hostname; my %replacements = ( _DATETIME_ => $datetime, _USER_ => $user, + _HOST_ => $host, ); if ($opt_o) { open $out, '>', $opt_o or die "Can't create $opt_o: $!\n"; print STDERR "opened file $opt_o for output\n" if $opt_d; + $replacements{_OUTPUTFILE_} = $opt_o; } else { open $out, '>&', STDOUT; print STDERR "using STDOUT for output\n" if $opt_d; + $replacements{_OUTPUTFILE_} = 'STDERR'; } if ($opt_m) { @@ -113,6 +118,7 @@ foreach my $r (sort {$a<=>$b} keys %snippets) { my $f = $s->[0]; print STDERR " snippet $n from file $f\n" if $opt_d; open $in, '<', $f or die "Can't open $f: $!\n"; + $replacements{_SNIPPETFILE_} = $f; print $dep " \\\n $f" if $opt_M; while (<$in>) { chomp; diff --git a/src/tools/test/Snippets.plt b/src/tools/test/Snippets.plt index eaf1a263a..56b7d21ad 100644 --- a/src/tools/test/Snippets.plt +++ b/src/tools/test/Snippets.plt @@ -1,10 +1,12 @@ #!/usr/bin/env perl use File::Path; +use Sys::Hostname; -use Test::More tests => 26; +use Test::More tests => 29; my $user = $ENV{LOGNAME} || $ENV{USER} || $ENV{USERNAME}; +my $host = hostname; mkdir "a$$"; mkdir "b$$"; @@ -76,7 +78,13 @@ is assemble("a$$/10_a", "b$$/2_a", "a$$/15_a", "b$$/09_a"), '2 09 10 15', "rank # Builtin macros mksnip('a', '30_a', '_USER_'); +mksnip('a', '30_b', '_OUTPUTFILE_'); +mksnip('a', '30_c', '_SNIPPETFILE_'); +mksnip('a', '30_d', '_HOST_'); is assemble("a$$/30_a"), "$user", "builtin macro _USER_"; +is assemble("a$$/30_b"), "out$$", "builtin macro _OUTPUTFILE_"; +is assemble("a$$/30_c"), "a$$/30_c", "builtin macro _SNIPPETFILE_"; +is assemble("a$$/30_d"), "$host", "builtin macro _HOST_"; # User macros mksnip('b', '35_a', 'Line _M1_'); From 214edd42d19689f212dcc87276b1671375dac324 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sun, 11 Oct 2015 00:50:50 -0500 Subject: [PATCH 9/9] Minor RULES_EXPAND changes Added support for a _PATTERN variable which calls $(wildcard) and prepends .. and the other SRC_DIRS automatically. Use $(ECHO) instead of @echo so make -s works silently. Added another meta-rule to create .d files when make needs one. --- configure/RULES_EXPAND | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/configure/RULES_EXPAND b/configure/RULES_EXPAND index 69636db19..0f43db347 100644 --- a/configure/RULES_EXPAND +++ b/configure/RULES_EXPAND @@ -34,20 +34,32 @@ expand_clean: ASSEMBLE_TOOL ?= $(PERL) $(TOOLS)/assembleSnippets.pl define COMMON_ASSEMBLY_template -$(COMMON_DIR)/$1: $($1_SNIPPETS) - @echo "Assembling file $$@ from snippets" +$1_SNIPPETS += $$(foreach dir, .. $$(SRC_DIRS), \ + $$(wildcard $$(dir)/$$($1_PATTERN))) +$(COMMON_DIR)/$1: $$($1_SNIPPETS) + $(ECHO) "Assembling common file $$@ from snippets" @$(RM) $1 - $(ASSEMBLE_TOOL) -o $1 $($1_SNIPPETS) + $(ASSEMBLE_TOOL) -o $1 $$^ @$(MV) $1 $$@ endef $(foreach asy, $(COMMON_ASSEMBLIES), \ $(eval $(call COMMON_ASSEMBLY_template,$(strip $(asy))))) define ASSEMBLY_template -$1: $($1_SNIPPETS) - @echo "Assembling file $$@ from snippets" +$1_SNIPPETS += $$(foreach dir, .. $$(SRC_DIRS), \ + $$(wildcard $$(dir)/$$($1_PATTERN))) +$1: $$($1_SNIPPETS) + $(ECHO) "Assembling file $$@ from snippets" @$(RM) $$@ - $(ASSEMBLE_TOOL) -o $$@ $($1_SNIPPETS) + $(ASSEMBLE_TOOL) -o $$@ $$^ endef $(foreach asy, $(ASSEMBLIES), \ $(eval $(call ASSEMBLY_template,$(strip $(asy))))) + +define ASSEMBLY_DEP_template +$1$(DEP): + @echo $1: > $$@ +endef +$(foreach asy, $(sort $(COMMON_ASSEMBLIES) $(ASSEMBLIES)), \ + $(eval $(call ASSEMBLY_DEP_template,$(strip $(asy))))) +