Iso-anonymizer.pl - Revision history

Tim at 18:43, 11 June 2016

2016-06-11T18:43:10Z

Tim at 10:50, 15 May 2016

2016-05-15T10:50:34Z

← Older revision		Revision as of 10:50, 15 May 2016
Line 191:		Line 191:

	=head1 TODO		=head1 TODO
		⚫	- reliably replace network address by networks with consistent netmasks
	- define test cases
		⚫	(currently all networks are reduced to a /32 netmask)
⚫	- reliably ~~replase~~ network address by networks with consistent netmasks
⚫	currently all networks are reduced to a /32 netmask
	- optimize speed

	=head1 AUTHOR		=head1 AUTHOR

Tim at 10:49, 15 May 2016

2016-05-15T10:49:23Z

← Older revision		Revision as of 10:49, 15 May 2016
Line 4:		Line 4:
	# iso-anonymizer.pl		# iso-anonymizer.pl
	# run like this:		# run like this:
	# ./iso-anonymizer.pl -~~txt_subst_file~~=/var/tmp/strings.txt [-net="192.168.0.0/16"] <config-file1 config-file2 ...>		# ./iso-anonymizer.pl -txt-subst-file=/var/tmp/strings.txt [-net="192.168.0.0/16"] <config-file1 config-file2 ...>
	# -------------------------------------------------------------------------------------------		# -------------------------------------------------------------------------------------------
	require 5.006_000; # Needed for NetAddr::IP and file handler		require 5.006_000; # Needed for NetAddr::IP and file handler
Line 26:		Line 26:
	my $ano_suffix = '.anonymized';		my $ano_suffix = '.anonymized';

			sub create_string_subst_hash {
			my $txt_subst_file_local = shift;
		⚫	open( my $txt_file, $txt_subst_file_local ) or croak "Unable to open $txt_subst_file_local: $!\n";
		⚫	while (my $line = <$txt_file>) {
		⚫	chomp ($line);
		⚫	$anonymized_text{$line} = $ano_txt;
		⚫	# adding separator chars (_-) contained in pattern again:
		⚫	if ($line =~ /.*?([\_\-])$/) { $anonymized_text{$line} .= $1; }
		⚫	if ($line =~ /^([\_\-]).*?/) { $anonymized_text{$line} = $1 . $anonymized_text{$line}; }
		⚫	++$ano_txt;
		⚫	}
		⚫	close ($txt_file);
			return;
			}
	sub _in_range { return 0 <= $_[0] && $_[0] <= 255; }		sub _in_range { return 0 <= $_[0] && $_[0] <= 255; }

Line 78:		Line 92:
	}		}
	});		});
	# obfuscating text		if (defined($txt_subst_file)) { # obfuscating text
	my $regex_all_texts = join("\|", map {quotemeta} keys %anonymized_text);		my $regex_all_texts = join("\|", map {quotemeta} keys %anonymized_text);
	$line =~ s/($regex_all_texts)/$anonymized_text{$1}/go;		$line =~ s/($regex_all_texts)/$anonymized_text{$1}/go;
			}
	print $ofh $line;		print $ofh $line;
	}		}
Line 88:		Line 103:
	my $start_time = time();		my $start_time = time();
	my $query = CGI->new;		my $query = CGI->new;
	if (defined(param("-~~txt_subst_file~~"))) { $txt_subst_file = param("-~~txt_subst_file~~"); } else { print ("error: no ~~txt_subst_file~~ specified\n"); exit 1; }		if (defined(param("-txt-subst-file"))) { $txt_subst_file = param("-txt-subst-file"); } else { print ("error: no -txt-subst-file param specified\n"); exit 1; }
	if (defined(param("-net"))) { $net = param("-net"); } else { $net = "10.0.0.0/8"; print ("no net specified, using default net $net\n"); }		if (defined(param("-net"))) { $net = param("-net"); } else { $net = "10.0.0.0/8"; print ("no net specified, using default net $net\n"); }
			if (defined($txt_subst_file)) { &create_string_subst_hash($txt_subst_file); }

⚫	open( my $txt_file, $~~txt_subst_file~~ ) or croak "Unable to open $~~txt_subst_file~~: $!\n";
⚫	while (my $line = <$txt_file>) {
⚫	chomp ($line);
⚫	$anonymized_text{$line} = $ano_txt;
⚫	# adding separator chars (_-) contained in pattern again:
⚫	if ($line =~ /.*?([\_\-])$/) { $anonymized_text{$line} .= $1; }
⚫	if ($line =~ /^([\_\-]).*?/) { $anonymized_text{$line} = $1 . $anonymized_text{$line}; }
⚫	++$ano_txt;
⚫	}
⚫	close ($txt_file);

	my $total_filesize = 0;		my $total_filesize = 0;
Line 126:		Line 131:

	=head1 SYNOPSIS		=head1 SYNOPSIS
	./iso-anonymizer.pl -~~txt_subst_file~~=/var/tmp/strings.txt [-net="192.168.0.0/16"] <config-file1 config-file2 ...>		./iso-anonymizer.pl -txt-subst-file=/var/tmp/strings.txt [-net="192.168.0.0/16"] <config-file1 config-file2 ...>

	=head1 DESCRIPTION		=head1 DESCRIPTION
Line 169:		Line 174:

	=head1 EXAMPLES		=head1 EXAMPLES
	./iso-anonymizer.pl -net="172.20.0.0/21" -~~txt_subst_file~~=/var/tmp/strings.txt /var/tmp/firewall17.cfg /var/tmp/router9.cfg		./iso-anonymizer.pl -net="172.20.0.0/21" -txt-subst-file=/var/tmp/strings.txt /var/tmp/firewall17.cfg /var/tmp/router9.cfg

	tim@lacantha:$ sudo perl iso-anonymizer.pl -~~txt_subst_file~~=strings.txt /var/tmp/netscreen1.cfg		tim@lacantha:$ sudo perl iso-anonymizer.pl -txt-subst-file=strings.txt /var/tmp/netscreen1.cfg
	no net specified, using default net 10.0.0.0/8		no net specified, using default net 10.0.0.0/8
	anonymizing: /var/tmp/netscreen1.cfg ... result file = /var/tmp/netscreen1.cfg.anonymized		anonymizing: /var/tmp/netscreen1.cfg ... result file = /var/tmp/netscreen1.cfg.anonymized
Line 182:		Line 187:
	# turn binary .Fc dump into ascii (only necessary if you do not already have an ascii dump):		# turn binary .Fc dump into ascii (only necessary if you do not already have an ascii dump):
	pg_restore >dbdump.sql		pg_restore >dbdump.sql
	sudo perl iso-anonymizer.pl -~~txt_subst_file~~=/var/tmp/strings.txt /var/tmp/files-to-anonymize/*		sudo perl iso-anonymizer.pl -txt-subst-file=/var/tmp/strings.txt /var/tmp/files-to-anonymize/*
	psql --set ON_ERROR_STOP=on targetdb <dbdump.sql		psql --set ON_ERROR_STOP=on targetdb <dbdump.sql

Tim at 10:17, 15 May 2016

2016-05-15T10:17:00Z

← Older revision		Revision as of 10:17, 15 May 2016
Line 104:		Line 104:
	my $total_filesize = 0;		my $total_filesize = 0;
	# treating all params not starting with - as files to anonymize		# treating all params not starting with - as files to anonymize
			# do not re-anonymize files with .anonymized extension and do not anonymize binary files
	foreach my $file ($query->param) {		foreach my $file ($query->param) {
	if ($file !~ /^-/ && $file !~ /.*?$ano_suffix$/) {		if ($file !~ /^-/ && $file !~ /.*?$ano_suffix$/ && -T $file) {
	$total_filesize += -s $file;		$total_filesize += -s $file;
	print ("anonymizing: $file ... ");		print ("anonymizing: $file ... ");
	&anonymize($file, $net, $file . $ano_suffix);		&anonymize($file, $net, $file . $ano_suffix);
	print ("result file = $file~~.anonymized~~\n");		print ("result file = $file$ano_suffix\n");
	}		}
	}		}
Line 120:		Line 121:
	printf(" (total %.2f MB, %.2f Mbytes/second).\n", $total_filesize/1000000, $total_filesize/$duration/1000000);		printf(" (total %.2f MB, %.2f Mbytes/second).\n", $total_filesize/1000000, $total_filesize/$duration/1000000);

	</pre>
	== Documentation ==
	<pre>
	=head1 NAME		=head1 NAME

Tim at 09:49, 15 May 2016

2016-05-15T09:49:15Z

← Older revision		Revision as of 09:49, 15 May 2016
Line 120:		Line 120:
	printf(" (total %.2f MB, %.2f Mbytes/second).\n", $total_filesize/1000000, $total_filesize/$duration/1000000);		printf(" (total %.2f MB, %.2f Mbytes/second).\n", $total_filesize/1000000, $total_filesize/$duration/1000000);

			</pre>
			== Documentation ==
			<pre>
	=head1 NAME		=head1 NAME

Tim at 09:44, 15 May 2016

2016-05-15T09:44:34Z

← Older revision		Revision as of 09:44, 15 May 2016
Line 59:		Line 59:
	find_ipaddrs($line, sub {		find_ipaddrs($line, sub {
	my($ipaddr, $orig) = @_;		my($ipaddr, $orig) = @_;
	# print ("found " . $ipaddr . " (orig=$orig)\n");
	if ($orig =~ /^2[45][0258]\./) { # found netmask (assuming IPs starting with 24x.* and 25x.* are netmasks)		if ($orig =~ /^2[45][0258]\./) { # found netmask (assuming IPs starting with 24x.* and 25x.* are netmasks)
	return $anonymized_ip{$orig} if exists $anonymized_ip{$orig};		return $anonymized_ip{$orig} if exists $anonymized_ip{$orig};
Line 80:		Line 79:
	});		});
	# obfuscating text		# obfuscating text
	# foreach my $orig (keys %anonymized_text) { my $subst = $anonymized_text{"$orig"}; $line =~ s/$orig/$subst/g; }
	my $regex_all_texts = join("\|", map {quotemeta} keys %anonymized_text);		my $regex_all_texts = join("\|", map {quotemeta} keys %anonymized_text);
	$line =~ s/($regex_all_texts)/$anonymized_text{$1}/go;		$line =~ s/($regex_all_texts)/$anonymized_text{$1}/go;

Tim: Die Seite wurde neu angelegt: „

 #! /usr/bin/perl -w # ------------------------------------------------------------------------------------------- # iso-anonymizer.pl # run like this:  #…“

2016-05-15T09:43:38Z

Die Seite wurde neu angelegt: „<pre> #! /usr/bin/perl -w # ------------------------------------------------------------------------------------------- # iso-anonymizer.pl # run like this: #…“

New page

<pre>
#! /usr/bin/perl -w
# -------------------------------------------------------------------------------------------
# iso-anonymizer.pl
# run like this:
# ./iso-anonymizer.pl -txt_subst_file=/var/tmp/strings.txt [-net="192.168.0.0/16"] <config-file1 config-file2 ...>
# -------------------------------------------------------------------------------------------
require 5.006_000; # Needed for NetAddr::IP and file handler
require Exporter;
use strict;
use warnings;
use CGI qw(:standard);
use NetAddr::IP;
use Carp;
use Time::HiRes qw(time tv_interval); # for exact recording of script execution time

my ($cfg_file, $line);
our @ISA = qw(Exporter);
my $infile;
my $txt_subst_file;
my $net;
my $outfile;
my %anonymized_ip;
my %anonymized_text;
my $ano_txt = "IsoAAAA"; # starting pattern - needs to be alpha chars only for incrementing to work
my $ano_suffix = '.anonymized';

sub _in_range { return 0 <= $_[0] && $_[0] <= 255; }

sub find_ipaddrs (\$&) {
my($r_text, $callback) = @_;
my $addrs_found = 0;
my $regex = qr<(\d+)\.(\d+)\.(\d+)\.(\d+)(\/\d\d?)?>;

$$r_text =~ s{$regex}{
my $orig_match = join '.', $1, $2, $3, $4;
if (defined($5) && $5 ne '') { $orig_match .= '/32'; }
if ((my $num_matches = grep { _in_range($_) } $1, $2, $3, $4) == 4) {
$addrs_found++;
my $ipaddr = NetAddr::IP->new($orig_match);
$callback->($ipaddr, $orig_match);
} else {
$orig_match;
}
}eg;
return $addrs_found;
}

sub anonymize {
my $infile = shift;
my $net = shift;
my $outfile = shift;
my $ip = NetAddr::IP->new("$net");

open( my $ifh, $infile ) or croak "Unable to open $infile: $!\n";
open( my $ofh, ">$outfile" ) or croak "Unable to open $outfile: $!\n" ;

while (my $line = <$ifh>) {
find_ipaddrs($line, sub {
my($ipaddr, $orig) = @_;
# print ("found " . $ipaddr . " (orig=$orig)\n");
if ($orig =~ /^2[45][0258]\./) { # found netmask (assuming IPs starting with 24x.* and 25x.* are netmasks)
return $anonymized_ip{$orig} if exists $anonymized_ip{$orig};
$anonymized_ip{$orig} = "255.255.255.255"; # changing all netmask to /32 to avoid invalid cidrs
return $anonymized_ip{$orig};
} elsif ($orig eq '0.0.0.0') { # leave /0 netmask alone
return $ipaddr->addr;
} else {
my $netmask = '';
if ($orig =~ /(.+?)\/32$/) {
$orig = $1;
$netmask = '/32';
}
return $anonymized_ip{$orig} . $netmask if exists $anonymized_ip{$orig};
# if found ip has not yet an anonymous equivalent in hash - create new ip
++$ip;
$anonymized_ip{$orig} = $ip->addr;
return $anonymized_ip{$orig} . $netmask;
}
});
# obfuscating text
# foreach my $orig (keys %anonymized_text) { my $subst = $anonymized_text{"$orig"}; $line =~ s/$orig/$subst/g; }
my $regex_all_texts = join("|", map {quotemeta} keys %anonymized_text);
$line =~ s/($regex_all_texts)/$anonymized_text{$1}/go;
print $ofh $line;
}
close ($ifh); close ($ofh); return;
}

my $start_time = time();
my $query = CGI->new;
if (defined(param("-txt_subst_file"))) { $txt_subst_file = param("-txt_subst_file"); } else { print ("error: no txt_subst_file specified\n"); exit 1; }
if (defined(param("-net"))) { $net = param("-net"); } else { $net = "10.0.0.0/8"; print ("no net specified, using default net $net\n"); }

open( my $txt_file, $txt_subst_file ) or croak "Unable to open $txt_subst_file: $!\n";
while (my $line = <$txt_file>) {
chomp ($line);
$anonymized_text{$line} = $ano_txt;
# adding separator chars (_-) contained in pattern again:
if ($line =~ /.*?([\_\-])$/) { $anonymized_text{$line} .= $1; }
if ($line =~ /^([\_\-]).*?/) { $anonymized_text{$line} = $1 . $anonymized_text{$line}; }
++$ano_txt;
}
close ($txt_file);

my $total_filesize = 0;
# treating all params not starting with - as files to anonymize
foreach my $file ($query->param) {
if ($file !~ /^-/ && $file !~ /.*?$ano_suffix$/) {
$total_filesize += -s $file;
print ("anonymizing: $file ... ");
&anonymize($file, $net, $file . $ano_suffix);
print ("result file = $file.anonymized\n");
}
}

# Generating statistics
my @ki=keys(%anonymized_ip);
my @kt=keys(%anonymized_text);
my $duration = time() - $start_time;
print("Anonymized " . $#ki . " ip addresses and " . $#kt . " strings in " . sprintf("%.1f",$duration) . " seconds");
printf(" (total %.2f MB, %.2f Mbytes/second).\n", $total_filesize/1000000, $total_filesize/$duration/1000000);

=head1 NAME

iso-anonymizer.pl - replace IP addresses with anonymized IPs as well as text with anonymized text in plain text files

=head1 SYNOPSIS
./iso-anonymizer.pl -txt_subst_file=/var/tmp/strings.txt [-net="192.168.0.0/16"] <config-file1 config-file2 ...>

=head1 DESCRIPTION

This is a module for
a) replacing IP addresses in plain text with anonymized equivalents from
the network range supplied.

b) replacing strings in a file with anonymized strings

Input is a number of ASCII files (all parameters not starting with -)
IP addresses as well as strings are replaced one-for-one throughout
all text files, so once an IP address has an anonymized equivalent,
it stays that way.

This is useful if you need to use production configuration data for testing.
E.g. from firewalls but do not want to expose the production data on a
test system. This way you can protect an organization's
identity at the same time.

Caveats:
- currently only implemented for IPv4
- beware of anonymizing common strings; e.g. "INT" when handling database dumps is part of keyword CONSTRAINT
use slightly longer strings like "INT_" instead

Params:
- The network range used for replacement, is set to "10.0.0.0/8" if omitted.
- For each file <infile> supplied an anonymized file called
<infile>.anonymized is created.

The second argument is a network address, which should be given in
CIDR notation, and really represents a range of IP addresses from
which we can draw from while doing the IP address substitutions (Note
that the use of NetAddr::IP means that we will never overflow this
range - but it will wrap around if we increment it enough). Using an
RFC1918 private address range is a good idea.

Note that the script tries to handle network addresses so that
network address and netmask (both given in 255.255.255.x notation
as well as a.b.c.d/xy notation) will match by simply setting
all netmasks to /32.

=head1 EXAMPLES
./iso-anonymizer.pl -net="172.20.0.0/21" -txt_subst_file=/var/tmp/strings.txt /var/tmp/firewall17.cfg /var/tmp/router9.cfg

tim@lacantha:$ sudo perl iso-anonymizer.pl -txt_subst_file=strings.txt /var/tmp/netscreen1.cfg
no net specified, using default net 10.0.0.0/8
anonymizing: /var/tmp/netscreen1.cfg ... result file = /var/tmp/netscreen1.cfg.anonymized
Anonymized 20197 ip addresses and 150 strings in 31.1 seconds (0.46 Mbytes/second).
tim@lacantha:~$

Anonymizing a whole (ASCII) Postgresql database:
# creating a dump of the database:
pg_dump
# turn binary .Fc dump into ascii (only necessary if you do not already have an ascii dump):
pg_restore >dbdump.sql
sudo perl iso-anonymizer.pl -txt_subst_file=/var/tmp/strings.txt /var/tmp/files-to-anonymize/*
psql --set ON_ERROR_STOP=on targetdb <dbdump.sql

=head1 TODO
- define test cases
- reliably replase network address by networks with consistent netmasks
currently all networks are reduced to a /32 netmask
- optimize speed

=head1 AUTHOR
Tim Purschke E<lt>tmp@cactus.deE<gt>

=head1 COPYRIGHT AND LICENSE
Copyright (C) 2016 by Cactus eSecurity GmbH

=head1 SEE ALSO
Behind the door

=cut
</pre>