#!/bin/perl5
# -*- perl -*-
##############################################################################
#                                                                            #
#  Copyright 1998 DJ Delorie <dj@delorie.com>                                #
#                                                                            #
#  Distributed under the terms of the GNU General Public License             #
#                                                                            #
#  For full instructions and copyright information, see                      #
#        http://www.delorie.com/gnu/gnu2html/                                #
#                                                                            #
##############################################################################
#
# 1997-Oct-01 dj@delorie.com	Initial version
# 1998-Mar-14 dj@delorie.com	Minor bug fixes
# 1999-Feb-07 dj@delorie.com	load ftp functions, download bundles
#

# Set these to reflect your local GNU mirror.
$ftp_host = "prep.ai.mit.edu";
$ftp_user = "anonymous";
$ftp_password = "gnu2html@";
$ftp_directory = "/pub/gnu";

# If you set these, the gnudocs.ini file is retrieved each time
$ini_host = "ftp.delorie.com";
$ini_user = "anonymous";
$ini_password = "gnu2html@";
$ini_directory = "/pub/gnu";
$ini_file = "gnudocs.ini";

# Set these to the directories where everything lives
$default_ini = "gnudocs.ini";
$temp = "/tmp";
$web = "/home/web/gnu/docs";
#$ftp_cache = "/tmp/gnu2html-cache"; # if set, it's permanent

# Set these to the program to use to convert the files
$man = "man2html";
$texi = "texi2html";
$texiopts = "-menu";

# Set these if you want additional download bundles
#$dl_src = 1;			# to make *-texi.tgz
#$dl_html = 1;			# to make *-html.tgz
#$texi2dvi = "texi2dvi -b";	# makes *-dvi.tgz
#$dvips = "dvips";		# makes *-ps.tgz (requires $texi2dvi)
#$texi2ps = "texi2ps";		# makes *-ps2up.tgz (from djgpp)
#$texi2ps_opts = "-v -f 8 -2 -m 36";

# for debugging
#$sh_opts = "-x";
#$show_lines = 0;

##############################################################################
#                                                                            #
#                  Nothing below this line should be changed                 #
#                                                                            #
##############################################################################

#$sh_opts = "set $sh_opts;" if $sh_opts;

use IO::Socket;

$ftp_debug_level = 0;

$| = 1;

# Just check the ftp server for new files
if ($ARGV[0] eq "-check") {
    $check = 1;
}

# Just check the ftp server for new files
if ($ARGV[0] eq "-getnew") {
    if (! $ftp_cache ) {
	print STDERR "$0: can't use -getnew without $ftp_cache\n";
	exit 1;
    }
    $getnew = 1;
}

if ($ftp_cache) {
    $ftp_tmp = $ftp_cache;
} else {
    $ftp_tmp = "$temp/gnu2html.ftp.$$";
}
mkdir($ftp_tmp, 0700) unless -d $ftp_tmp;

#&ftp_debug(2); # to debug both sessions

if ($ini_host) {
    print "Fetching ftp://$ini_user\@$ini_host$ini_directory/$ini_file\n"
	unless $check;
    exit 0 if &ftp_open($ini_host); #'
    &ftp_login($ini_user, $ini_password) && exit 1; #'
    &ftp_cwd($ini_directory) && exit 1; #'
    &ftp_get($ini_file, "/tmp/gnudocs.ini.$$"); #'
    $default_ini = "/tmp/gnudocs.ini.$$";
    $delete_ini_later = 1;
    &ftp_close(); #'
}

#&ftp_debug(2); # to debug the gnu session

print "Connecting to ftp://$ftp_user\@$ftp_host$ftp_directory\n"
    unless $check;

exit 1 if &ftp_open($ftp_host);

&ftp_login($ftp_user, $ftp_password) && exit 1;

&ftp_cwd($ftp_directory) && exit 1;

&ftp_dir_open("*") && exit 1;

open(HIS, "$web/.ftp.history");
while (<HIS>) {
    s/[\r\n]+$//;
    ($name, $timestamp, $ftpstamp) = split(' ', $_, 3);
    $ftp_timestamp{$name} = $timestamp;
    $old_ftpstamp{$name} = $ftpstamp;
}

$subdir = '';
while (<$ftp_data>) {
    if (/^(\S+):[\r\n]+$/) {
	$subdir = "$1/";
    }
    next unless /^-/;
    s/[ \t]+/ /g;
    s/[\r\n]+$//;
    ($ftpstamp, $name) = /\S+ \S+ \S+ (.+) (\S+)/;
    $new_ftpstamp{$name} = $ftpstamp;
    $ftp_subdir{$name} = $subdir;

    if ($check || $getnew) {
	if ($new_ftpstamp{$name} ne $old_ftpstamp{$name}) {
	    print "$name `$new_ftpstamp{$name}' vs `$old_ftpstamp{$name}'\n"
		if $check;
	    if ($getnew) {
		push(@needed_files, $name);
	    }
	}
    }
}

&ftp_dir_close();

if ($getnew) {
    for $f (@needed_files) {
	if ( ! -f "$ftp_tmp/$f") {
	    print "Getting $f\n";
	    &need_ftp_file($f);
	}
    }
}

exit 0 if $check || $getnew;

rename("$web/.ftp.history", "$web/.ftp.history.old");

$now = time;
open(HIS, "> $web/.ftp.history");
for $f (sort keys %new_ftpstamp) {
    if ($old_ftpstamp{$f} ne $new_ftpstamp{$f}) {
	$ftp_timestamp{$f} = $now;
    }
    print HIS "$f $ftp_timestamp{$f} $new_ftpstamp{$f}\n";
}
close(HIS);

sub need_ftp_file {
    local($f) = @_;
    return if -f "$ftp_tmp/$f";
    $subdir = $ftp_subdir{$f};
    if ($f =~ m@(.*)/.*@) {
	print "mkdir $ftp_tmp/$1\n";
	mkdir("$ftp_tmp/$1", 0755);
    }
    print "ftp: getting $subdir$f\n";
    &ftp_get("$subdir$f", "$ftp_tmp/$f.tmp");
    rename("$ftp_tmp/$f.tmp", "$ftp_tmp/$f");
}

#-----------------------------------------------------------------------------

$web = &fix($web);


$ENV{"PATH"} = "/usr/local/bin:" . $ENV{"PATH"};


$ini = shift;
$ini = $default_ini unless $ini;

open(INI, $ini);
@lines = <INI>;
close(INI);

for $_ (@lines) {

    s/\#.*//;
    s/[\r\n]+$//;
    next unless /\S/;

    print "\nLINE: $_\n" if $show_lines;

    last if /^\@exit/;

    if (/^x (\S+)(\s+(\S+))?/) {
	$arc = $1;
	$files = $3;

	$tgz = "$arc";
	$dep = $arc;
	if ( $ftp_timestamp{"$arc.tar.gz"} ) {
	    $tgz = "$arc.tar.gz";
	    $dep = "$arc.tar.gz";
	}
	if ( $ftp_timestamp{"$arc.tgz"} ) {
	    $tgz = "$arc.tgz";
	    $dep = "$arc.tgz";
	}

	next if &prepare_directory($arc, $dep);

	&need_ftp_file($tgz);
	system "$sh_opts gzcat $ftp_tmp/$tgz | (cd $prev_tmp/tarx; tar xfB - $files)";
	$first = 1;
    }

    elsif (/^d (\S+) (.*)/) {
	$arc = $1;
	@deps = split(' ', $2);

	next if &prepare_directory($arc, @deps);
	$first = 0;
    }

    elsif (/^S/) {
	$just_subindex{$arclist[$#arclist]} = 1;
    }

    elsif (/^NI/) {
	close H;
    }

    elsif ( ! $arc) {
    }

    elsif (/^n (.*)/) {
	$arcname = $1;
    }

    elsif (/^z (\S+)(.*)/) {
	$tgz = "$1";
	$files = $2;
	exit unless $prev_tmp;
	&need_ftp_file($tgz);
	if ($tgz =~ /\.tar\.gz/) {
	    system "$sh_opts gzcat $ftp_tmp/$tgz | (cd $prev_tmp/tarx; tar xfB - $files)";
	} else {
	    system "$sh_opts cp $ftp_tmp/$tgz $prev_tmp";
	}
    }

    elsif (/^C (.*)/) {
	print H "</multicol>\n" if $multicol;
	$multicol = $1;
	print H "<multicol cols=$multicol>\n";
    }

    elsif (/^f (\S+)/) {
	system("cd $prev_tmp; $sh_opts find tarx -type f -name '$1' -exec cp {} . \\; -print");
    }

    elsif (/^! (.+)/) {
	system "cd $prev_tmp; $sh_opts $1";
    }

    elsif (/^w (\S+)(\s+(.+))?/) {
	$desc = $3;
	$desc = $1 unless $desc;
	if ( ! -f "$prev_tmp/$1") {
	    system("cd $prev_tmp; $sh_opts find tarx -type f -name '$1' -exec cp {} . \\; -print");
	}
	&move_html("$1");
	$h = $1;
	if ($first) {
	    $t = $arcname;
	    $t =~ s/-/ /g;
	} else {
	    $t = $desc;
	}
	print H "<li> <a href=\"$h\">$t</a>\n";
	&check("$arcdir/$h");
	$first = 0;
    }

    elsif (/^([tT]) (\S+)(\s+(.+))?/) {
	$file = $2;
	$name = $4;
	if ( ! -f "$prev_tmp/$file") {
	    system("cd $prev_tmp; $sh_opts find tarx -type f -name '$file' -exec cp {} . \\; -print");
	}

	# some .texi files don't have @bye, TeX can't deal with that,
	# so we add one just in case
	open(TI, ">>$prev_tmp/$file");
	print TI "\@bye\n";
	close(TI);

	&before_builds();

	$split = "-split_node";
	$split = "" if $1 eq "T";
	system "cd $prev_tmp; $sh_opts $texi $texiopts $split $file";
	&move_html("*.html");

	($base = $file) =~ s/\.te?xi?(nfo)?$//;

	if ($texi2dvi) {
	    system("cd $prev_tmp; $sh_opts $texi2dvi $file");
	    if ($dvips) {
		system("cd $prev_tmp; $sh_opts $dvips -o $base.ps $base");
	    }
	}

	if ($texi2ps) {
	    system("cd $prev_tmp; $sh_opts $texi2ps $texi2ps_opts $file > $base.ps2");
	}

	&after_builds();


	$h = $file;
	$t = $file;
	$h =~ s/\.te?xi?(nfo)?$/_toc.html/;
	if ($first) {
	    $t = $arcname;
	    $t =~ s/-/ /g;
	} else {
	    $t =~ s/\.te?xi?(nfo)?$//;
	    $t = $name if $name;
	}
	print H "<li> <a href=\"$h\">$t</a>\n";
	&check("$arcdir/$h");
	$first = 0;
    }

    elsif (/^m (\S+)(\s+(.+))?/) {
	$name = $3;
	if ( ! -f "$prev_tmp/$1") {
	    system("cd $prev_tmp; $sh_opts find tarx -type f -name '$1' -exec cp {} . \\; -print");
	}
	&before_builds();
	system "cd $prev_tmp; $sh_opts $man < $1 > $1.html";
	&move_html("$1.html");
	&after_builds();
	$h = $1;
	$t = $1;
	$h .= ".html";
	$t =~ s/\.man$/.1/;
	$t =~ s/\.([^-]+)/($1)/;
	if ($first) {
	    $t = $arcname;
	    $t =~ s/-/ /g;
	} else {
	    $t = $name if $name;
	}
	print H "<li> <a href=\"$h\">$t</a>\n";
	&check("$arcdir/$h");
	$first = 0;
    }
}

sub prepare_directory {
    local($arctmp, @deps) = @_;
    $arc = $arctmp;

    $arcname = $arc;
    $arcdir = $arc;
    ($base, $version) = $arc =~ /^(.*)-(\d+(\..*)*)$/;
    $arcdir =~ s/-(\d+(\..*)*)$//;

    &cleanup();

    push (@arclist, $arc);

    if (! -f "$web/$arcdir/index.html") {
	$stale = 1;
    } else {
	$stale = 0;
	for $d (@deps) {
	    if ((stat("$web/$arcdir/index.html"))[9] < $ftp_timestamp{$d}) {
		$stale = 1;
	    }
	}
    }

    if ( -f "$web/$arcdir/.version" ) {
	open(V, "$web/$arcdir/.version");
	($oldversion,$ftpstamp) = split(' ',<V>);
	close(V);
	if ($oldversion ne $version ||
	    ($ftpstamp && $ftpstamp ne $new_ftpstamp{$arc})) {
	    $stale = 1;
	}
    } elsif (-d "$web/$arcdir") {
	open(V, "> $web/$arcdir/.version");
	print V "$version $new_ftpstamp{$arc}";
	close(V);
    }

    if (! $stale) {
	&skip($arc);
	$arc = "";
	return 1;
    }

    print "\n" if $skipped;
    print "\n";
    $skipped = 0;

    $prev_tmp = "$temp/gnu2html.$$";
    mkdir($prev_tmp, 0700);
    mkdir("$prev_tmp/tarx", 0700);

    $prev_arc = $arcdir;

    system "mkdir", "-p", "$web/$arcdir" unless -d "$web/$arcdir";
    open(H, "> $web/$arcdir/index.html");
    print H &cat_with_title("subpage-top.html", $arc);
    print H "<ul>\n";

    open(V, "> $web/$arcdir/.version");
    print V $version;
    close(V);

    return 0;
}

&cleanup;

sub cleanup {
    if ($prev_tmp) {
	print H "</multicol>\n" if $multicol;
	print H "</ul>\n";
	print H &cat_with_title("subpage-bottom.html", "");
	close(H);

	$do_html = $dl_dvi = $dl_ps = $dl_ps2 = 0;

	if ($dl_src) {
	    @src = ();
	    system "echo $prev_tmp; ls $prev_tmp";
	    opendir(PT, $prev_tmp);
	    for $f (sort readdir PT) {
		next if $f eq "core";
		print "check $f - bf=$built_files{$f}\n";
		push(@src, $f) if (-f "$prev_tmp/$f"
				   && $f !~ /\.(html|dvi|ps2?|gz)$/
				   && ! $built_files{$f});
		$do_html= 1 if $f =~ /\.html$/;
		$dl_dvi = 1 if $f =~ /\.dvi$/;
		$dl_ps  = 1 if $f =~ /\.ps$/;
		$dl_ps2 = 1 if $f =~ /\.ps2$/;
	    }
	    $src = join(' ', @src);
	    if ($src) {
		system("cd $prev_tmp; $sh_opts tar cvf - $src | gzip -9 > $web/$prev_arc/$prev_arc-texi.tgz");
	    }
	}
	%built_files = ();
	if ($dl_html && $do_html) {
	    system("cd $web/$prev_arc; $sh_opts tar cf - *.html | gzip -9 > $prev_arc-html.tgz");
	}
	system("cd $prev_tmp; $sh_opts tar cf - *.dvi | gzip -9 > $web/$prev_arc/$prev_arc-dvi.tgz") if $dl_dvi;
	system("cd $prev_tmp; $sh_opts tar cf - *.ps | gzip -9 > $web/$prev_arc/$prev_arc-ps.tgz") if $dl_ps;
	system("cd $prev_tmp; $sh_opts tar cf - *.ps2 | gzip -9 > $web/$prev_arc/$prev_arc-ps2up.tgz") if $dl_ps2;

	system "$sh_opts /bin/rm -rf $prev_tmp";
	$prev_tmp = "";
	$multicol = 0;
	@texinfo_files = ();
    }
}

sub before_builds {
    %source_files = ();
    opendir(PT, $prev_tmp);
    for $f (readdir PT) {
	$source_files{$f} = 1;
    }
}

sub after_builds {
    opendir(PT, $prev_tmp);
    for $f (readdir PT) {
	$built_files{$f} = 1 unless $source_files{$f};
    }
}

print "\n" if $skipped;
print "\n";
open(I, "> $web/index.html");

open(ITOP, "index-top.html");
print I while <ITOP>;
close(ITOP);

print I "<ul>\n";

print "Scanning...";
$len = 11;
for $arc (sort @arclist) {

    $arcdir = $arc;
    $arcdir =~ s/-(\d+(\..*)*)$//;
    open(A, "$web/$arcdir/index.html");

    $len += length($arcdir)+2;
    if ($len > 75) {
	print "\n  ";
	$len = length($arcdir) + 4;
    }
    print "  $arcdir";


    if ($just_subindex{$arc}) {
	while (<A>) {
	    next if /<li>/;
	}
	($title) = m/>(.*)<\/a>/;
	$title = $arc unless $title;
	print I "<li><a href=\"$arcdir/index.html\">$title</a>\n";
    } elsif ($arc =~ /^[A-Z]/) {
	print I "\n<b>";
	while (<A>) {
	    next unless /<li>/ || /<\/?multicol/;
	    if (/<\/?multicol/) {
		print I;
		next;
	    }
	    s@href=\"@href=\"$arcdir/@; #"
	    print I;
	}
	print I "</b></p>\n";
    } else {
	print I "\n";
	$count = 0;
	    $links_in_index = 0;
	while (<A>) {
	    next unless /<li>/ || /<\/?multicol/;
	    $count ++;
	    $links_in_index ++;
	    print I "<ul>" if $count == 2;
	    if (/<\/?multicol/) {
		print I;
		next;
	    }
	    s@href=\"@href=\"$arcdir/@; #"
	    print I;
	}
	print I "</ul>" if $count >= 2;
	if ($links_in_index == 0) {
	    print "Warning: no links in $arcdir\n";
	}
    }

    close(A);
}

print "\n";

print I "\n</ul>\n";

open(IBOT, "index-bottom.html");
print I while <IBOT>;
close(IBOT);

close(I);

system "/bin/rm -rf $ftp_tmp" unless $ftp_cache;
&ftp_close();

if ($delete_ini_later) {
    unlink $default_ini;
}

#############################################################################

sub check {
    local($f) = @_;
    if ( ! -f "$web/$f" ) {
	print "\n*** Error: can't find file $web/$f\n\n";
    }
}

sub fix {
    local($d) = @_;
    $d = `cd $d; pwd`;
    $d =~ s/[\r\n]+//;
    return $d;
}

sub move_html {
    local($f) = @_;
    system("cd $prev_tmp; $sh_opts chmod +w $f; mv -f $f $web/$arcdir");
}

sub skip {
    local($arc) = @_;
    print "\n" unless $skipped;
    if (!$skipped) {
	print "skipping...";
	$len = 11;
    }
    $len += length($arc)+2;
    if ($len > 75) {
	print "\n  ";
	$len = 4+length($arc)+1;
    }
    print "  $arc";
    $skipped = 1;
}

sub cat_with_title {
    local($f,$t) = @_;
    local($rv) = "";
    open(CAT, $f);
    while (<CAT>) {
	s/%s/$t/g;
	$rv .= $_;
    }
    close(CAT);
    return $rv;
}

#############################################################################

sub ftp_debug {
    $ftp_debug_level = $_[0];
}

sub ftp_command {
    my ($command) = @_;
    if (defined $command) {
	print "ftp > $command\n" if $ftp_debug_level;
	print $ftp_control "$command\n";
    }
    do {
	$ftp_response = <$ftp_control>;
	print "ftp < $ftp_response" if $ftp_debug_level;
    } until $ftp_response =~ /^\d\d\d /;
    $ftp_response =~ s/[\r\n]$//;
    if ($ftp_response =~ /^[4-9]/) {
	print STDERR "FTP Error: $ftp_response\n";
	return 1;
    }
    return 0;
}

sub ftp_open {
    my ($site) = @_;
    $ftp_save_site = $site;
    $ftp_control = IO::Socket::INET->new(PeerAddr => $site,
					 PeerPort => 21,
					 Proto => "tcp");
    &ftp_command() && return 1;
    return 0;
}

sub ftp_login {
    my ($user, $password) = @_;
    $ftp_save_user = $user;
    $ftp_save_password = $password;
    &ftp_command("USER $user") && return 1;
    if ($ftp_response =~ /^331/) {
	&ftp_command("PASS $password") && return 1;
    }
    &ftp_command("TYPE I");
    return 0;
}

sub ftp_cwd {
    my ($dir) = @_;
    $ftp_save_dir = $dir;
    return &ftp_command("CWD $dir");
}

sub ftp_open_data_port {
    return 0 if $ftp_data_port_ready;
    &ftp_command("PASV");
    my ($h1,$h2,$h3,$h4,$p1,$p2) = $ftp_response =~ /\((\d+),(\d+),(\d+),(\d+),(\d+),(\d+)\)/;
    $p1 = $p1 * 256 + $p2;
    print "PASV: $h1.$h2.$h3.$h4 port $p1\n" if $ftp_debug_level > 1;
    $ftp_data = IO::Socket::INET->new(PeerAddr => "$h1.$h2.$h3.$h4",
				      PeerPort => $p1,
				      Proto => "tcp");

    if (! defined $ftp_data) {
	return 1;
    }
    $ftp_data_port_ready = 1;
    return 0;
}


sub ftp_close_data_port {
    close $ftp_data;
    $ftp_data_port_ready = 0;
}

sub ftp_get {
    my ($remote_name, $local_file) = @_;
    open(FTP_F, ">$local_file") || return 1;
    if (&ftp_open_data_port()) {
	print "ftp pasv failed: $ftp_response\n";
	if ($ftp_response =~ /timeout/i) {
	    print "ftp: attempting to re-establish the connection\n";
	    &ftp_open($ftp_save_site) && return 1;
	    &ftp_login($ftp_save_user, $ftp_save_password) && return 1;
	    &ftp_cwd($ftp_save_dir) && return 1;
	    &ftp_open_data_port() && return 1;
	} else {
	    return 1;
	}
    }
    &ftp_command("RETR $remote_name") && return 1;
    print FTP_F $ftp_buffer while read($ftp_data, $ftp_buffer, 4096) > 0;
    close(FTP_F);
    &ftp_close_data_port();
    return &ftp_command();
}

sub ftp_close {
    &ftp_command("quit");
    close $ftp_control;
    return;
}

sub ftp_dir_open {
    my ($options) = @_;
    &ftp_open_data_port();
    $options = " $options" if $options;
    &ftp_command("LIST$options");
    $ftp_data_port_ready = 0;
}

sub ftp_dir_close {
    &ftp_close_data_port();
    return &ftp_command();
}

1;