#!/usr/bin/perl -w
# terpub - tool for downloading instrument approach plates
# and other terminal procedures publications

use strict;
use Symbol;
use LWP::Simple;
use LWP::UserAgent;
use Data::Dumper;

my $main_chart_host="aeronav.faa.gov";
my $execfile='c:\strawberry\perl\bin\perl.exe';

sub usage {
  print <<EoF;

Terpub allows you to download large groups of terminal procedures
publications (TPPs).  This is much less laborious (and less
error-prone) compared to going directly to $main_chart_host and
clicking on the links one by one.  This is useful for keeping
your "electronic flight case" up-to-date.

Note that TPPs include instrument approach plates, arrival procedures
(STARs), departure procedures, takeoff minimums, alternate minimums,
airport diagrams, hot spot notices, LAHSO notices, et cetera.

Terpub uses an old-school "command line interface" (as opposed to a
mouse-click interface).

Usage:
  terpub [options] [identifiers]

  Each identifier given on the command line is checked as follows, in
  order of precedence:

    *) If it is the two-letter code for a state, all TPPs for that
    state will be downloaded.

    *) If it is of the form CITY,ST then all TPPs for the matching
    city (or cities) will be downloaded.

    *) Otherwise, if it is an FAA or ICAO airport identifier, all TPPs
    for that airport will be downloaded.

    *) Otherwise, if it matches the name of a city in any state(s),
    all matching TPPs will be downloaded.  Beware that "Springfield"
    exists in 10 different states, and if you don't specify the state
    then all of them will get downloaded.

  All names and identifiers are case-insensitive.

  If no identifiers are given, we just rebuild the index.html file.

Examples:
  Download all the the instrument procedures for Portland
  (Oregon) International airport.
          terpub kpdx

  Download all of the instrument approaches for Portland
  (Oregon) International and the entire state of Washington:
          terpub kpdx WA

Process and products:

  0) If no -cycle was specified on the command line, we attempt to
  figure out what the current /cycle number/ is.

  0) We create (in the current working directory) a subdirectory named
  after the "cycle number", e.g. ./tpp-1607/.

  0) We fetch the metadata for this cycle, and store it in the
  aforementioned tpp-cycle directory.  If it is already on disk, we do
  not re-fetch it.

  1) We fetch the selected TPPs and store them on disk in the
  tpp-cycle directory.

  For each TPP, if it is already on disk we do not re-fetch it.

  2) For TPPs that have recently changed, we fetch the comparison
  pdf, if available.  This document highlights the changes.

  3) We build an "index.html" file in the tpp-cycle directory.  This
  gets rebuilt every time, even if the -nofetch option is in effect.

  The index.html file lists all available TPPs, and hotlinking to
  where they are -- or would be -- stored on the local machine.  All
  TPPs are listed, no matter whether they have been downloaded on this
  run, downloaded previously, or not yet downloaded at all.  TPPs that
  are not (yet) locally present are indexed in red.

  In the index, one of the columns contains the airport identifier(s).
  If the FAA identifier (e.g. JFK) can be inferred from ICAO
  identifier (e.g. KJFK) by dropping the first letter, then only the
  ICAO identifier is shown.  If there is no ICAO identifier,
  (e.g. 39N) then only the FAA identifier is shown.  If both exist but
  they do not correspond, then both are shown, (e.g. PAFA :: FAI).  In
  all cases, if the FAA identifier is shown, it is italicized.

Options:
   -help
    Print this message

   -older
    Note that toward the end of each period, two cycles of the
    procedures are available: the older ones that will soon expire,
    and the newer ones that will soon come into effect.  We download
    the newer ones by default.  You can specify -older if you want
    the older ones.

   -verbose
    Print progress messages as we go along.

   -cycle NNNN
    Fetch files from cycle NNNN no matter what the FAA thinks
    is current.

   -nofetch
    Don't actually fetch any TPPs.  Useful for rebuilding the
    index.html file.  Also useful with -v to see what would have been
    fetched.

Other publications:
  VFR Sectional Aeronautical Charts can be downloaded via:
    http://www.aeronav.faa.gov/index.asp?xml=aeronav/applications/VFR/chartlist_sect
  IFR low-altitude enroute charts can be downloaded via:
    http://www.aeronav.faa.gov/index.asp?xml=aeronav/applications/IFR/chartlist_enroute

Other tools:

  You may wish to impose two TPPs on each side of a piece of paper.
  With two-sided printing that gives 4 TPPs per sheet of paper.
  Letter paper (8.5 x 11) works OK.  Legal paper (8.5 x 14) leaves a
  nice margin, making it easy to clip the plates onto your clipboard.
  Useful tools include:
    A tool for combining .pdf files:
      pdftk a.pdf b.pdf c.pdf cat output stuff.pdf
    An imposition program:
      java -jar bookbinder3.0.jar
    A more sophisticated, powerful imposition program:
      podofoimpose stuff.pdf stuff-duplex.pdf Booklet-USletter-duplex.plan
    Various imposition plans (for podofo) can be found at:
      http://www.av8n.com/computer/prepress/

Portability: Terpub uses standard Perl features and should run under
  just about any operating system, including Linux and Windows.  If
  your perl version is more than a year out of date, you should go to
  www.perl.org and download the latest version.  If your perl version
  is only slightly out of date, you can run cpan and give it the
  "upgrade" command.  In any case, you may need to fetch the
  LWP::Simple library module.  You can do that via:
        cpan LWP::Simple

  On a windows machine, if you observe that "terpub.pl -h" does
  nothing, and generally all perl scripts act like they were called
  with no arguments, it is a sign that your perl installation is
  defective.  There is a problem with the "file type associations".
  You may be able to fix it using the command:
        FTYPE pl_auto_file=$execfile %1 %*

Non-copyright:
  This software is in the public domain.

Instructions, warnings, and disclaimers:
  This is free software.  It is in the alpha (experimental) state.  It
  is made available as a means to communicate ideas.  You are
  instructed to evaluate it.  Incautious reliance on experimental
  software could create a hazardous situation.  There is *no*
  warranty; not even for merchantability or fitness for a particular
  purpose.
EoF

}

################################
# Status, including bugs and creeeping features:
#     *) Having index.html all in one file is convenient for
#       searching etc., but the file is so big that the
#       browser has trouble coping with it.  Maybe we would
#       be better off splitting it along state boundaries,
#       or along the established "bound volume" boundaries.
#     *) City mode needs polishing:
#       -- squeeze out punctuation
#       -- allow regular expressions, so that at least New.York
#        would match "New York".
#       -- accept abbreviations, if there are no exact matches.
#     *) Should replace tree-walk by hash-lookup based on airport ident.
#     *) Index file should go in the cycle-directory, not the current
#       working directory.
#     *) Index should indicate which files are actually on disk.
#       One option would be to provide hotlinks if and only if the
#       file is actually on disk.
#     *) Index might provide a means to download the files, by
#       clicking on some sort of button that links to a CGI script.
#       Perhaps checkboxes followed by "download all checked items"


# Global variables:
my $verbosity = 0;
my $really = 1;
my %todo = ();
my %todo2 = ();         # things to fetch, but OK if not available
my @idargs = ();
my @states = ();
my %allstate = ();      # the full record for each state, indexed
                        # by two-letter state code
my $Q = '"';

# The main data structure looks like this:
# 1) The universe contains an ordered list of state-objects.
# 2) Each state-object contains an ordered list of city-objects.
# 3) Each city-object contains an ordered list of airport-objects.
# 4) Each airport-object contains an ordered list of chart-objects.

# A chart-object corresponds to what the FAA calls a <record>.

# We use the word chart to refer to any TPP, even ones that
# are in text format (not just graphical chart format).

# We prefer the word "cycle" whereas the FAA prefers the word
# "version" to refer to the successive numbered release cycles.

# We use the word "enroll" to mean scheduling a chart to be
# fetched.  This involves appending it to %todo or %todo2.

sub scan_meta {
  my ($fname, $cycle_number) = @_;
  my $from_edate;
  my $to_edate;
  my $old_delim = $/;
  $/ = '>';
  my $inch = Symbol::gensym;
  open ($inch, '<', $fname)
    || die "Could not open input metafile '$fname'\n";

  my $cur_state;
  my $cur_city;
  my $cur_airport;
  my %tmp_chart;

  my $cur_apt_ident;
  my $cur_icao_ident;
  my @cities = ();
  my @apt_names = ();

#??  my %cities_by_state = ();
#??  my %states_by_city = ();

  my $inrecord = 0;
  my %interested = ();
# These are the fields that can occur in a hash
# representing a given chart:
  for my $field ('chart_code', 'chart_name', 'pdf_name', 'useraction',
                'record') {
    $interested{$field} = 1;
  }

  liner: while (my $line = <$inch>) {
    #xx print "$line\n";

# chart record stuff:
    if ($inrecord) {
      if ($line =~ m'^<[^/]') {
        next liner;     # a beginning; not interesting
      }
      if ($line =~ m'</') {
        my $inner = $`;         # inner HTML, before ending tag
        my $endof = $';         # entity of ending tag
        $endof = substr($endof, 0, length($endof)-1); # remove trailing >
        if ($interested{$endof}) {
          if ($endof eq 'record') {
            $inrecord = 0;
            push @{$cur_airport->{'charts'}}, {%tmp_chart};
            next liner;
          }
          $tmp_chart{$endof} = $inner;
        }
      } else {
        die "inrecord: not a beginning or an ending\n";
      }
      next liner;
    }

    if ($line =~ m'<digital_tpp') {
      my $rest = $';
      if ($rest =~ m'>') {
        my $settings = $`;
        if ($settings =~ m'\bcycle="([^"]*)"') {
          if ($cycle_number != $1) {
            die "File cycle $1 does not match requested $cycle_number\n";
          }
        } else {
          die "No cycle= within the <digital_tpp> element\n";
        }

        if ($settings =~ m'\bfrom_edate="([^"]*)"') {
          $from_edate = $1;
        } else {
          die "No from_edate= within the <digital_tpp> element\n";
        }

        if ($settings =~ m'\bto_edate="([^"]*)"') {
          $to_edate = $1;
        } else {
          die "No to_edate= within the <digital_tpp> element\n";
        }


      } else {
        die "Unclosed '<digital_tpp ...' element\n";
      }
    }
    if ($line =~ m'^<record>') {
      $inrecord = 1;
      %tmp_chart = (city => $cur_city);
      next liner;
    }

### airport stuff
    if ($line =~ m'^<airport_name ') {
      $cur_airport = {};                # ref to new anonymous hash
      push @{$cur_city->{'airports'}}, $cur_airport;
      $cur_airport->{'charts'} = [];
      if ($line =~ m'ID="([^"]*)"') {
        my $id = $1;
        $cur_airport->{'id'} = $id;
      } else {
        die "Airport without an ID?!\n";
      }
      if ($line =~ m'apt_ident="([^"]*)"') {
        $cur_airport->{faa_ident} = $1;
      }
      if ($line =~ m'icao_ident="([^"]*)"') {
        $cur_airport->{icao_ident} = $1;
      }
      next liner;
    }

### city stuff
    if ($line =~ m'^<city_name ') {
      $cur_city = {};              # ref to new anonymous hash
      push @{$cur_state->{'cities'}}, $cur_city;
      if ($line =~ m'ID="([^"]*)"') {
        my $id = $1;
        $cur_city->{'id'} = $id;
        my $ali_str = $id;
        $ali_str =~ s"[ \t]*[(]"/";
        $ali_str =~ s'[)]'';
        $cur_city->{'aliases'} = [split('/', $ali_str)];
      } else {
        die "City without an ID?!\n";
      }
      $cur_city->{'airports'} = [];
      next liner;
    }

### state stuff
    if ($line =~ m'^<state_code ') {
      $cur_state = {};              # ref to new anonymous hash
      push @states, $cur_state;
      if ($line =~ m'ID="([^"]*)"') {
        $cur_state->{'id'} = $1;
        $allstate{lc($1)} = $cur_state;
      } else {
        die "state with no ID?!\n";
      }

      if ($line =~ m'state_fullname="([^"]*)"') {
        $cur_state->{'fullname'} = $1;
      }
      $cur_state->{'cities'} = [];
      next liner;
    }
  }

# <airport_name ID="TINIAN INTL" military="N"
#       apt_ident="TNI" icao_ident="PGWT" alnum="6848">

  close $inch;
  $/ = $old_delim;
#xx  print Dumper({%allstate});
  return ($from_edate, $to_edate);
}

# Check return code via:  is_success($rslt)
# Special return code 299 means file already on disk.
# Special return code 298 means excused absence.
# Special return code 599 means file rename failed.
#
# possible usage:
#   url_to_file($url, $file, ok404=>'OK');
sub url_to_file {
  my $url = shift @_;
  my $file = shift @_;
  my %flags = @_;
  my $city = $flags{'city'};
  if ($verbosity > 2) {
    printf STDERR "url_to_file: %-30s  --> %s\n", $url, $file;
  }
  if (-r $file) {
    if ($verbosity) {
      print STDERR "... already on disk: '$file'";
      if ($city) {
        print STDERR "  # $city";
      }
      print STDERR "\n";
    }
    return 299;
  }
  $verbosity && print STDERR   "... downloading:     '$file'\n";
  my $ua = LWP::UserAgent->new;
  my $resp = $ua->get($url, ':content_file' => "$file.part");
  my $headers = $resp->{_headers};
  if ($verbosity > 2) {
    for my $key (sort keys %$resp) {
      print STDERR "$key -> $resp->{$key}\n";
    }
    print STDERR "-----------------\n";
    for my $key (sort keys %$headers) {
      print STDERR "$key -> $headers->{$key}\n";
    }
    print STDERR "=================\n";
  }
  my $rslt = $resp->{_rc};
  if (is_success($rslt)) {
    if ($verbosity > 1) {
      print STDERR "... success ($rslt) $headers->{'content-type'}\n";
    }
    if (rename("$file.part", $file)) {
      return $rslt;
    }
    print STDERR "... rename failed: '$file.part' --> '$file'\n";
    return 599;
  }
  if ($flags{ok404} && $rslt == 404) {
    $verbosity && print STDERR "... excused absence: '$file'\n";
    return 298;
  }
  printf STDERR "*** download failed: $rslt\n";
  printf STDERR "*** $resp->{_msg}\n";
  printf STDERR "*** URL was: $url\n";
  # should we remove the .part file?
  return $rslt;
}

sub write_index_html {
  my ($cycle_number, $from_edate, $to_edate) = @_;

  my $PageHeader = <<EoF;
      <head>
      <title>
      Instrument Procedures
      </title>
      </head>
      <body><html>
      <center>
      <table style="font-size:150%">
      <tr>
        <td>Cycle $cycle_number</td>
        <td>effective from</td>
        <td>$from_edate</td>
      </tr>
      <tr>
        <td></td>
        <td align="right">to</td>
        <td>$to_edate</td>
      </tr>
      </table>
      </center>
      <div align=right><a href="./">[Directory listing]</a></div>
      <center>
EoF

## Colgroup does not appreciably speed up the rendering,
## but does improve the appearance quite a bit.
  my $TableHeader = <<EoF;
    <table border=1>
        <colgroup>
        <col style="width:15em">
        <col style="width:15em">
        <col style="width:10em">
        <col style="width:3em">
        <col style="width:5em">
        <col style="width:18em">
      </colgroup>
      <tbody>
EoF

  my $ndx = Symbol::gensym;
  my $cycle_dir = "tpp-$cycle_number";
  my $fname = "$cycle_dir/index.html";
  open($ndx, '>', $fname)
      || die "unable to write index file '$fname' : $!\n";
  print $ndx $PageHeader;

  my $td = '<td style="padding-left:10px;padding-right:10px;">';
  my $nbsp = '&nbsp;';
  for my $state (@states) {
    print $ndx "<a name=$Q$state->{'id'}$Q></a>",
                "<h2>$state->{'fullname'}</h2>\n";
    print $ndx $TableHeader;
    for my $city (@{$state->{'cities'}}) {
      my $city_id = $city->{'id'};
# Experiments show <wbr> is definitely an improvement:
      $city_id =~ s'-'-<wbr>';
      $city_id =~ s'/'/<wbr>';
      my $last_city_id = '--none--';
      for my $airport (@{$city->{'airports'}}){
        my $ident;
        my $icao_ident = $airport->{'icao_ident'};
        my $faa_ident = $airport->{'faa_ident'};
        if ($icao_ident eq 'K' . $faa_ident
              || $icao_ident eq 'P' . $faa_ident) {
          $ident = $icao_ident;
        } elsif ($icao_ident eq '') {
          $ident = "<em>$faa_ident</em>";
        } else {
          $ident = "$icao_ident :: <em>$faa_ident</em>";
        }
        my $airport_id = $airport->{'id'};
        $airport_id =~ s'-'-<wbr>';
        $airport_id =~ s'/'/<wbr>';
        my $last_airport_id = '--none--';
        for my $chart (@{$airport->{'charts'}}){
          if ($city_id eq $last_city_id) {
            $city_id = '';
          }
          $last_city_id = $city_id;

          if ($airport_id eq $last_airport_id) {
            $airport_id = '';
            $ident = '';
          }
          $last_airport_id = $airport_id;

          print $ndx "<tr>\n";
          print $ndx "$td$city_id</td>\n";
          print $ndx "$td$airport_id</td>\n";
          print $ndx "$td$ident</td>\n";
          my $act = $chart->{'useraction'};
          $act =~ s/[ \t]*//g;
          if ($act eq '') {
            $act = $nbsp;
          }
          my $pdf = $chart->{'pdf_name'};
          my $exists = 'style="color:red"';
          if (-r "$cycle_dir/$pdf") {
            $exists = '';
          }
          my $compare = $pdf;
          $compare =~ s/[.]PDF/_CMP.PDF/;
          my $newpatt = "<a href=$Q./compare_pdf/$compare$Q>C</a>";
# It is entirely OK if the following substitution fails;
# it just means that there were no changes.
          $act =~ s"C"$newpatt";
          print $ndx "$td$act</td>\n";
          print $ndx "$td$chart->{'chart_code'}</td>\n";
          my $chartname = $chart->{'chart_name'};
          $chartname =~ s'RWY 'RWY&nbsp;';
          print $ndx "$td<a href=$Q./$pdf",
                        "$Q $exists>$chartname</a></td>";

          print $ndx "<tr>\n";
        }
      }
    }
    print $ndx "</tbody></table>\n";
  }


# OK thats everything, lets terminate the table

    print $ndx <<EoF;
    </center>
    </html>
    </body>
EoF
    close($ndx);
}

sub enroll_whole_airport {
  my ($airport) = @_;
  for my $chart (@{$airport->{'charts'}}) {
    my $pdf = $chart->{'pdf_name'};
    my $act = $chart->{'useraction'};

    if ($act !~ m/D/) {
      $todo{$pdf} = 1;
    } else {
      my $cname = $chart->{'chart_name'};
      $verbosity && print "... skipping deleted: $cname ($pdf)\n";
    }

    if ($act =~ m/C/) {
      my $compare = $pdf;
      $compare =~ s/[.]PDF/_CMP.PDF/;
      $todo2{"compare_pdf/$compare"} = 1;
    }
  }
}

sub enroll_whole_city {
  my ($city) = @_;
  for my $airport (@{$city->{'airports'}}) {
    enroll_whole_airport($airport);
  }
}

sub enroll_whole_state {
  my ($state) = @_;
  for my $city (@{$state->{'cities'}}) {
    enroll_whole_city($city);
  }
}

# Scan all the args in @idargs and fetch
# the corresponding requested documents
sub scan_identifier_args {
  fetcher: while (@idargs) {
    my $idarg = shift @idargs;
    if ($idarg =~ ',$') {
      $idarg .= shift @idargs;
    }
    $idarg = lc($idarg);
    #### print STDERR "... $idarg ...\n";
    if (length($idarg) == 2) {
      #xx print STDERR "st: $idarg\n";
      if (exists $allstate{$idarg}) {
        enroll_whole_state($allstate{$idarg});
        next fetcher;
      }
      die "No match for state abbrev '$idarg'\n";
    }
    my $did1 = 0;

####
# handle "city,st"
    my @stuff = split(",", $idarg);
    if ((0+@stuff) == 2) {
      my $ct = $stuff[0];
      my $st = $stuff[1];
      if (exists $allstate{$st}) {
        my $state = $allstate{$st};
        for my $city (@{$state->{'cities'}}) {
          for my $alias (@{$city->{'aliases'}}) {
            $alias = lc($alias);
            if ($ct eq $alias) {
              enroll_whole_city($city);
              $did1++;
            }
          }
        }
        if ($did1) {
          next fetcher;         # success
        }
        die "No match for city '$ct' given arg '$idarg'\n";
      }
      die "No match for state '$st' given arg '$idarg'\n";
    }
####
# handle airport identifier:
    for my $state (@states) {
      for my $city (@{$state->{'cities'}}) {
        for my $airport (@{$city->{'airports'}}) {
          if ($idarg =~ m/^$airport->{'faa_ident'}$/i
           || $idarg =~ m/^$airport->{'icao_ident'}$/i) {
             $verbosity && print "Match airport: $idarg\n";
             enroll_whole_airport($airport);
             next fetcher;        # assume airport idents are unique
          }
        }
      }
    }
####
# handle stateless city:
    if (1) {                    # stateless city is the only remaining possibility
      my $ct = $idarg;
      for my $st (keys %allstate) {     # loop over all states
        my $state = $allstate{$st};     # get full record for this state
        ##### print STDERR "... $st ... $state ...\n";
        for my $city (@{$state->{'cities'}}) {
          for my $alias (@{$city->{'aliases'}}) {
            $alias = lc($alias);
            if ($ct eq $alias) {
              # the following is amusing for finding all states that
              # have a city of the given name:
              if (0) {
                print STDERR "... $ct, $st ...\n";
              }
              enroll_whole_city($city);
              $did1++;
            }
          }
        }
        ### $did1 || print STDERR "No match for city '$ct' in state '$st' given arg '$idarg'\n";
      }
      $did1 || die "No match for city '$ct' in any state given arg '$idarg'\n";
    }

#xxxx    printf STDERR "Sorry, not implemented: stateless city: '$idarg'\n";

  }     # end loop over idargs
}

main: {

  my $cycle_number;
  my $newer = 1;
  my $index_only = 0;          # don't actually download the files, just list them
  my $id_type="ICAO";          # use ICAO airport ID codes as default
  my $proto = 'https';         # use SSL if at all possible

  my $preindex_url = "$proto://www.faa.gov/air_traffic/flight_info/"
                        . 'aeronav/digital_products/dtpp/search/';

  argloop: while(@ARGV) {
     my $arg = shift @ARGV;
     $arg =~ tr/A-Z/a-z/;          #down shift the case on everything
     my $opt = $arg;
     if ($opt =~ s/^-?-//) {
       if ($opt eq 'index-only') {
           $index_only = 1;
       } elsif ($opt eq 'old' || $opt eq 'older') {
           $newer = -1;
       } elsif ($opt eq 'help' || $opt eq 'h') {
           usage();
           exit 0;
       } elsif ($opt eq 'verbose' || $opt eq 'v') {
           $verbosity++;
       } elsif ($opt eq 'n' || $opt eq 'nofetch') {
           $really = 0;
       } elsif ($opt eq 'cycle') {
         my $cycle = shift @ARGV;
         if (! defined $cycle) {
           print STDERR "-cycle requires an argument (cycle number)\n";
           exit 1;
         }
         $cycle_number = 0 + $cycle;
       } else {
         die "Unrecognized option '$arg'\n";
       }
       next argloop;
     }
     push @idargs, $arg;
  }

# end of arg loop

# Lets go to the main site and parse out the revision number
# and the effective period.

  my $from_edate;
  my $to_edate;

  if (!$cycle_number) {
    if ($verbosity) {
      $verbosity && print STDERR "Fetching preindex using url: $preindex_url\n";
    }

    my $content = get $preindex_url;
    if (!defined $content) {
      print STDERR "unable to fetch contents of index page.\n";
      print STDERR "URL was: $preindex_url\n";
      exit 1;
    }

    get_ed: for my $line (split("\n", $content)) {
       chomp $line;

       my $patt = '(https?)://aeronav.faa.gov/'
                . 'd-tpp/([0-9]+)/xml_data/d-tpp_Metafile.xml';
       if ($line =~ m/$patt/) {
         $cycle_number = $2;
         my $metafile_url = $&;
         if ($verbosity) {
           print STDERR "Found cycle_number $cycle_number ... $1\n";
         }
       }
    }
  }

  if (!$cycle_number) {
    die "Failed to obtain cycle number from $preindex_url\n";
  }

# Here with the appropriate cycle number.

# typical chart file:
#       http://aeronav.faa.gov/d-tpp/1607/00430il11l.pdf
# uppercase works too, and is easier on the eyes:
#       http://aeronav.faa.gov/d-tpp/1607/00430IL11L.pdf

# standard metafile (but was briefly missing):
#       http://aeronav.faa.gov/d-tpp/1012/xml_data/d-TPP_Metafile.xml
# older metafile:
#       http://aeronav.faa.gov/d-tpp/1012/d-TPP_Metafile.xml
#
# Another (emergency) way to obtain: metafile "d-TPP_Metafile.xml"
#       http://www.aeronav.faa.gov/upload_313-d/terminal/DDTPPE_201607.zip

  my $cycle_dir = "tpp-$cycle_number";
  if (! -d "$cycle_dir") {
    print STDERR "Creating cycle directory '$cycle_dir'\n";
    mkdir ("$cycle_dir")
      || die "Failed to mkdir '$cycle_dir' : $!\n";
  }

  my $didsome = 0;
  my $metafile = "$proto://$main_chart_host/d-tpp/"
        . "$cycle_number/xml_data/d-TPP_Metafile.xml";

  my $sts = url_to_file($metafile, "$cycle_dir/d-TPP_Metafile.xml");
  if ($sts == 500) {    ## try again
    $proto = 'http';
    print STDERR "*** Warning: trying downgrade to $proto\n";
    $metafile = "$proto://$main_chart_host/d-tpp/"
        . "$cycle_number/xml_data/d-TPP_Metafile.xml";
    $sts = url_to_file($metafile, "$cycle_dir/d-TPP_Metafile.xml");
  }
  if (!is_success($sts)) {
    die"Bad sts: $sts\n";
  }
  if (is_success($sts) && $sts != 298 && $sts != 299) {
    $didsome++;
  }

  ($from_edate, $to_edate) =
    scan_meta("$cycle_dir/d-TPP_Metafile.xml", $cycle_number);

  scan_identifier_args;
#xxxxx  print Dumper(@states);

  # Here with a list of things to fetch in %todo and %todo2

  my $didgripe = 0;
  if (! -d "$cycle_dir") {
    mkdir ("$cycle_dir")
      || die "Failed to mkdir '$cycle_dir' :: $!\n";
  }
  if (! -d "$cycle_dir/compare_pdf") {
    mkdir "$cycle_dir/compare_pdf";
    # should check for errors here
  }

  if ($really) {
    charter: for my $doit (keys %todo) {
      my $sts = url_to_file("$proto://$main_chart_host/d-tpp/$cycle_number/$doit",
                 "$cycle_dir/$doit");
      if (is_success($sts) && $sts != 298 && $sts != 299) {
        $didsome++;
      }
      if (is_success($sts)) {
        next charter;
      }
      if ($sts == 500 && !$didsome) {
        $proto = 'http';
        print STDERR "*** Warning: trying downgrade to $proto\n";
        $sts = url_to_file("$proto://$main_chart_host/d-tpp/$cycle_number/$doit",
                 "$cycle_dir/$doit");
        if (is_success($sts) && $sts != 298 && $sts != 299) {
          $didsome++;
        }
        if (is_success($sts)) {
          next charter;
        }
        $didgripe++;
      }
    }
    if ($didgripe){
      print STDERR "xxx Some charts failed to fetch.\n";
    }

##    url_to_file('http://localhost/asdfasdf', 'barf', ok404=>'OK');

    for my $doit (keys %todo2) {
      url_to_file("$proto://$main_chart_host/d-tpp/$cycle_number/$doit",
         "$cycle_dir/$doit", ok404=>'OK')
        || exit 3;
    }
  } elsif ($verbosity) {
    if (%todo) {
      print "Would have fetched:\n";
      print join("\n", sort keys %todo), "\n";
    }
    if (%todo2) {
      print "Also would have tried to fetch:\n";
      print join("\n", sort keys %todo2), "\n";
    }
  }

## Calculate the index /last/, so it can be smart
## about the existence of newly-downloaded files:
  write_index_html($cycle_number, $from_edate || '???', $to_edate || '!!!');
}