untrusted comment: verify with openbsd-69-base.pub RWQQsAemppS46GVrctnNxZ6LM0UIEBt2dQOANLoodljGVdciP2yAEMcMpj4gV1PnHhXBWh3k9Y2HLIEftJi4MdIhFDK2mQn4eQQ= OpenBSD 6.9 errata 021, November 9, 2021: rpki-client(8) should handle CA misbehaviours as soft-errors. Apply by doing: signify -Vep /etc/signify/openbsd-69-base.pub -x 021_rpki.patch.sig \ -m - | (cd /usr/src && patch -p0) And then rebuild and install rpki-client and openrsync cd /usr/src/usr.sbin/rpki-client make obj make make install cd /usr/src/usr.bin/rsync make obj make make install Index: usr.sbin/rpki-client/Makefile =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/Makefile,v retrieving revision 1.21 diff -u -p -u -r1.21 Makefile --- usr.sbin/rpki-client/Makefile 1 Apr 2021 16:04:48 -0000 1.21 +++ usr.sbin/rpki-client/Makefile 6 Nov 2021 18:09:25 -0000 @@ -3,9 +3,9 @@ PROG= rpki-client SRCS= as.c cert.c cms.c crl.c encoding.c gbr.c http.c io.c ip.c log.c \ main.c mft.c mkdir.c output.c output-bgpd.c output-bird.c \ - output-csv.c output-json.c parser.c repo.c roa.c rrdp.c rrdp_delta.c \ - rrdp_notification.c rrdp_snapshot.c rsync.c tal.c validate.c \ - x509.c + output-csv.c output-json.c parser.c print.c repo.c roa.c rrdp.c \ + rrdp_delta.c rrdp_notification.c rrdp_snapshot.c rsync.c tal.c \ + validate.c x509.c MAN= rpki-client.8 LDADD+= -lexpat -ltls -lssl -lcrypto -lutil Index: usr.sbin/rpki-client/cert.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/cert.c,v retrieving revision 1.28 diff -u -p -u -r1.28 cert.c --- usr.sbin/rpki-client/cert.c 5 Mar 2021 17:15:19 -0000 1.28 +++ usr.sbin/rpki-client/cert.c 6 Nov 2021 18:09:46 -0000 @@ -1,5 +1,6 @@ /* $OpenBSD: cert.c,v 1.28 2021/03/05 17:15:19 claudio Exp $ */ /* + * Copyright (c) 2021 Job Snijders * Copyright (c) 2019 Kristaps Dzonsons * * Permission to use, copy, modify, and distribute this software for any @@ -46,25 +47,19 @@ struct parse { const char *fn; /* currently-parsed file */ }; -/* - * Wrapper around ASN1_get_object() that preserves the current start - * state and returns a more meaningful value. - * Return zero on failure, non-zero on success. - */ -static int -ASN1_frame(struct parse *p, size_t sz, - const unsigned char **cnt, long *cntsz, int *tag) -{ - int ret, pcls; +static ASN1_OBJECT *carepo_oid; /* 1.3.6.1.5.5.7.48.5 (caRepository) */ +static ASN1_OBJECT *mft_oid; /* 1.3.6.1.5.5.7.48.10 (rpkiManifest) */ +static ASN1_OBJECT *notify_oid; /* 1.3.6.1.5.5.7.48.13 (rpkiNotify) */ - assert(cnt != NULL && *cnt != NULL); - assert(sz > 0); - ret = ASN1_get_object(cnt, cntsz, tag, &pcls, sz); - if ((ret & 0x80)) { - cryptowarnx("%s: ASN1_get_object", p->fn); - return 0; - } - return ASN1_object_size((ret & 0x01) ? 2 : 0, *cntsz, *tag); +static void +cert_init_oid(void) +{ + if ((carepo_oid = OBJ_txt2obj("1.3.6.1.5.5.7.48.5", 1)) == NULL) + errx(1, "OBJ_txt2obj for %s failed", "1.3.6.1.5.5.7.48.5"); + if ((mft_oid = OBJ_txt2obj("1.3.6.1.5.5.7.48.10", 1)) == NULL) + errx(1, "OBJ_txt2obj for %s failed", "1.3.6.1.5.5.7.48.10"); + if ((notify_oid = OBJ_txt2obj("1.3.6.1.5.5.7.48.13", 1)) == NULL) + errx(1, "OBJ_txt2obj for %s failed", "1.3.6.1.5.5.7.48.13"); } /* @@ -85,6 +80,8 @@ append_ip(struct parse *p, const struct if (!ip_addr_check_overlap(ip, p->fn, p->res->ips, p->res->ipsz)) return 0; + if (res->ipsz >= MAX_IP_SIZE) + return 0; res->ips = reallocarray(res->ips, res->ipsz + 1, sizeof(struct cert_ip)); if (res->ips == NULL) @@ -104,6 +101,8 @@ append_as(struct parse *p, const struct if (!as_check_overlap(as, p->fn, p->res->as, p->res->asz)) return 0; + if (p->res->asz >= MAX_AS_SIZE) + return 0; p->res->as = reallocarray(p->res->as, p->res->asz + 1, sizeof(struct cert_as)); if (p->res->as == NULL) @@ -228,9 +227,9 @@ sbgp_sia_resource_entry(struct parse *p, const unsigned char *d, size_t dsz) { ASN1_SEQUENCE_ANY *seq; + ASN1_OBJECT *oid; const ASN1_TYPE *t; int rc = 0, ptag; - char buf[128]; long plen; if ((seq = d2i_ASN1_SEQUENCE_ANY(NULL, &d, dsz)) == NULL) { @@ -254,7 +253,7 @@ sbgp_sia_resource_entry(struct parse *p, p->fn, ASN1_tag2str(t->type), t->type); goto out; } - OBJ_obj2txt(buf, sizeof(buf), t->value.object, 1); + oid = t->value.object; t = sk_ASN1_TYPE_value(seq, 1); if (t->type != V_ASN1_OTHER) { @@ -268,21 +267,17 @@ sbgp_sia_resource_entry(struct parse *p, d = t->value.asn1_string->data; dsz = t->value.asn1_string->length; - if (!ASN1_frame(p, dsz, &d, &plen, &ptag)) + if (!ASN1_frame(p->fn, dsz, &d, &plen, &ptag)) goto out; - /* - * Ignore all but manifest and RRDP notify URL. - * Things we may see: - * - 1.3.6.1.5.5.7.48.5 (caRepository) - * - 1.3.6.1.5.5.7.48.10 (rpkiManifest) - * - 1.3.6.1.5.5.7.48.13 (rpkiNotify) - */ - if (strcmp(buf, "1.3.6.1.5.5.7.48.5") == 0) + if (carepo_oid == NULL) + cert_init_oid(); + + if (OBJ_cmp(oid, carepo_oid) == 0) rc = sbgp_sia_resource_carepo(p, d, plen); - else if (strcmp(buf, "1.3.6.1.5.5.7.48.10") == 0) + else if (OBJ_cmp(oid, mft_oid) == 0) rc = sbgp_sia_resource_mft(p, d, plen); - else if (strcmp(buf, "1.3.6.1.5.5.7.48.13") == 0) + else if (OBJ_cmp(oid, notify_oid) == 0) rc = sbgp_sia_resource_notify(p, d, plen); else rc = 1; /* silently ignore */ @@ -436,7 +431,7 @@ sbgp_asrange(struct parse *p, const unsi goto out; } if (!as_id_parse(t->value.integer, &as.range.min)) { - warnx("%s: RFC 3770 section 3.2.3.8 (via RFC 1930): " + warnx("%s: RFC 3779 section 3.2.3.8 (via RFC 1930): " "malformed AS identifier", p->fn); return 0; } @@ -449,7 +444,7 @@ sbgp_asrange(struct parse *p, const unsi goto out; } if (!as_id_parse(t->value.integer, &as.range.max)) { - warnx("%s: RFC 3770 section 3.2.3.8 (via RFC 1930): " + warnx("%s: RFC 3779 section 3.2.3.8 (via RFC 1930): " "malformed AS identifier", p->fn); return 0; } @@ -484,12 +479,12 @@ sbgp_asid(struct parse *p, const ASN1_IN as.type = CERT_AS_ID; if (!as_id_parse(i, &as.id)) { - warnx("%s: RFC 3770 section 3.2.3.10 (via RFC 1930): " + warnx("%s: RFC 3779 section 3.2.3.10 (via RFC 1930): " "malformed AS identifier", p->fn); return 0; } if (as.id == 0) { - warnx("%s: RFC 3770 section 3.2.3.10 (via RFC 1930): " + warnx("%s: RFC 3779 section 3.2.3.10 (via RFC 1930): " "AS identifier zero is reserved", p->fn); return 0; } @@ -564,7 +559,7 @@ sbgp_asnum(struct parse *p, const unsign goto out; break; default: - warnx("%s: RFC 3779 section 3.2.3.4: IPAddressOrRange: " + warnx("%s: RFC 3779 section 3.2.3.5: ASIdOrRange: " "want ASN.1 sequence or integer, have %s (NID %d)", p->fn, ASN1_tag2str(tt->type), tt->type); goto out; @@ -666,7 +661,7 @@ sbgp_assysnum(struct parse *p, X509_EXTE d = t->value.asn1_string->data; dsz = t->value.asn1_string->length; - if (!ASN1_frame(p, dsz, &d, &plen, &ptag)) + if (!ASN1_frame(p->fn, dsz, &d, &plen, &ptag)) goto out; /* Ignore bad AS identifiers and RDI entries. */ @@ -985,36 +980,29 @@ out: * is also dereferenced. */ static struct cert * -cert_parse_inner(X509 **xp, const char *fn, int ta) +cert_parse_inner(X509 **xp, const char *fn, const unsigned char *der, + size_t len, int ta) { int rc = 0, extsz, c; + int sia_present = 0; size_t i; X509 *x = NULL; X509_EXTENSION *ext = NULL; ASN1_OBJECT *obj; struct parse p; - BIO *bio = NULL; - FILE *f; *xp = NULL; - if ((f = fopen(fn, "rb")) == NULL) { - warn("%s", fn); + /* just fail for empty buffers, the warning was printed elsewhere */ + if (der == NULL) return NULL; - } - - if ((bio = BIO_new_fp(f, BIO_CLOSE)) == NULL) { - if (verbose > 0) - cryptowarnx("%s: BIO_new_file", fn); - return NULL; - } memset(&p, 0, sizeof(struct parse)); p.fn = fn; if ((p.res = calloc(1, sizeof(struct cert))) == NULL) err(1, NULL); - if ((x = *xp = d2i_X509_bio(bio, NULL)) == NULL) { + if ((x = *xp = d2i_X509(NULL, &der, len)) == NULL) { cryptowarnx("%s: d2i_X509_bio", p.fn); goto out; } @@ -1039,6 +1027,7 @@ cert_parse_inner(X509 **xp, const char * c = sbgp_assysnum(&p, ext); break; case NID_sinfo_access: + sia_present = 1; c = sbgp_sia(&p, ext); break; case NID_crl_distribution_points: @@ -1050,6 +1039,8 @@ cert_parse_inner(X509 **xp, const char * break; case NID_subject_key_identifier: break; + case NID_ext_key_usage: + break; default: /* { char objn[64]; @@ -1069,12 +1060,52 @@ cert_parse_inner(X509 **xp, const char * p.res->aia = x509_get_aia(x, p.fn); p.res->crl = x509_get_crl(x, p.fn); } + if (!x509_get_expire(x, p.fn, &p.res->expires)) + goto out; + p.res->purpose = x509_get_purpose(x, p.fn); /* Validation on required fields. */ + switch (p.res->purpose) { + case CERT_PURPOSE_CA: + if (p.res->mft == NULL) { + warnx("%s: RFC 6487 section 4.8.8: missing SIA", p.fn); + goto out; + } + if (p.res->asz == 0 && p.res->ipsz == 0) { + warnx("%s: missing IP or AS resources", p.fn); + goto out; + } + break; + case CERT_PURPOSE_BGPSEC_ROUTER: + p.res->pubkey = x509_get_pubkey(x, p.fn); + if (p.res->pubkey == NULL) { + warnx("%s: x509_get_pubkey failed", p.fn); + goto out; + } + if (p.res->ipsz > 0) { + warnx("%s: unexpected IP resources in BGPsec cert", + p.fn); + goto out; + } + if (sia_present) { + warnx("%s: unexpected SIA extension in BGPsec cert", + p.fn); + goto out; + } + if (ta) { + warnx("%s: BGPsec cert can not be a trust anchor", + p.fn); + goto out; + } + break; + default: + warnx("%s: x509_get_purpose failed in %s", p.fn, __func__); + goto out; + } + if (p.res->ski == NULL) { - warnx("%s: RFC 6487 section 8.4.2: " - "missing SKI", p.fn); + warnx("%s: RFC 6487 section 8.4.2: missing SKI", p.fn); goto out; } @@ -1110,25 +1141,13 @@ cert_parse_inner(X509 **xp, const char * goto out; } - if (p.res->asz == 0 && p.res->ipsz == 0) { - warnx("%s: RFC 6487 section 4.8.10 and 4.8.11: " - "missing IP or AS resources", p.fn); - goto out; - } - - if (p.res->mft == NULL) { - warnx("%s: RFC 6487 section 4.8.8: " - "missing SIA", p.fn); - goto out; - } if (X509_up_ref(x) == 0) - errx(1, "king bula"); + errx(1, "%s: X509_up_ref failed", __func__); p.res->x509 = x; rc = 1; out: - BIO_free_all(bio); if (rc == 0) { cert_free(p.res); X509_free(x); @@ -1138,19 +1157,20 @@ out: } struct cert * -cert_parse(X509 **xp, const char *fn) +cert_parse(X509 **xp, const char *fn, const unsigned char *der, size_t len) { - return cert_parse_inner(xp, fn, 0); + return cert_parse_inner(xp, fn, der, len, 0); } struct cert * -ta_parse(X509 **xp, const char *fn, const unsigned char *pkey, size_t pkeysz) +ta_parse(X509 **xp, const char *fn, const unsigned char *der, size_t len, + const unsigned char *pkey, size_t pkeysz) { EVP_PKEY *pk = NULL, *opk = NULL; struct cert *p; int rc = 0; - if ((p = cert_parse_inner(xp, fn, 1)) == NULL) + if ((p = cert_parse_inner(xp, fn, der, len, 1)) == NULL) return NULL; if (pkey != NULL) { @@ -1200,38 +1220,11 @@ cert_free(struct cert *p) free(p->aia); free(p->aki); free(p->ski); + free(p->pubkey); X509_free(p->x509); free(p); } -static void -cert_ip_buffer(struct ibuf *b, const struct cert_ip *p) -{ - io_simple_buffer(b, &p->afi, sizeof(enum afi)); - io_simple_buffer(b, &p->type, sizeof(enum cert_ip_type)); - - if (p->type != CERT_IP_INHERIT) { - io_simple_buffer(b, &p->min, sizeof(p->min)); - io_simple_buffer(b, &p->max, sizeof(p->max)); - } - - if (p->type == CERT_IP_RANGE) - ip_addr_range_buffer(b, &p->range); - else if (p->type == CERT_IP_ADDR) - ip_addr_buffer(b, &p->ip); -} - -static void -cert_as_buffer(struct ibuf *b, const struct cert_as *p) -{ - io_simple_buffer(b, &p->type, sizeof(enum cert_as_type)); - if (p->type == CERT_AS_RANGE) { - io_simple_buffer(b, &p->range.min, sizeof(uint32_t)); - io_simple_buffer(b, &p->range.max, sizeof(uint32_t)); - } else if (p->type == CERT_AS_ID) - io_simple_buffer(b, &p->id, sizeof(uint32_t)); -} - /* * Write certificate parsed content into buffer. * See cert_read() for the other side of the pipe. @@ -1239,16 +1232,14 @@ cert_as_buffer(struct ibuf *b, const str void cert_buffer(struct ibuf *b, const struct cert *p) { - size_t i; + io_simple_buffer(b, &p->expires, sizeof(p->expires)); + io_simple_buffer(b, &p->purpose, sizeof(p->purpose)); + io_simple_buffer(b, &p->talid, sizeof(p->talid)); + io_simple_buffer(b, &p->ipsz, sizeof(p->ipsz)); + io_simple_buffer(b, &p->asz, sizeof(p->asz)); - io_simple_buffer(b, &p->valid, sizeof(int)); - io_simple_buffer(b, &p->ipsz, sizeof(size_t)); - for (i = 0; i < p->ipsz; i++) - cert_ip_buffer(b, &p->ips[i]); - - io_simple_buffer(b, &p->asz, sizeof(size_t)); - for (i = 0; i < p->asz; i++) - cert_as_buffer(b, &p->as[i]); + io_simple_buffer(b, p->ips, p->ipsz * sizeof(p->ips[0])); + io_simple_buffer(b, p->as, p->asz * sizeof(p->as[0])); io_str_buffer(b, p->mft); io_str_buffer(b, p->notify); @@ -1257,36 +1248,7 @@ cert_buffer(struct ibuf *b, const struct io_str_buffer(b, p->aia); io_str_buffer(b, p->aki); io_str_buffer(b, p->ski); -} - -static void -cert_ip_read(int fd, struct cert_ip *p) -{ - - io_simple_read(fd, &p->afi, sizeof(enum afi)); - io_simple_read(fd, &p->type, sizeof(enum cert_ip_type)); - - if (p->type != CERT_IP_INHERIT) { - io_simple_read(fd, &p->min, sizeof(p->min)); - io_simple_read(fd, &p->max, sizeof(p->max)); - } - - if (p->type == CERT_IP_RANGE) - ip_addr_range_read(fd, &p->range); - else if (p->type == CERT_IP_ADDR) - ip_addr_read(fd, &p->ip); -} - -static void -cert_as_read(int fd, struct cert_as *p) -{ - - io_simple_read(fd, &p->type, sizeof(enum cert_as_type)); - if (p->type == CERT_AS_RANGE) { - io_simple_read(fd, &p->range.min, sizeof(uint32_t)); - io_simple_read(fd, &p->range.max, sizeof(uint32_t)); - } else if (p->type == CERT_AS_ID) - io_simple_read(fd, &p->id, sizeof(uint32_t)); + io_str_buffer(b, p->pubkey); } /* @@ -1295,39 +1257,40 @@ cert_as_read(int fd, struct cert_as *p) * Always returns a valid pointer. */ struct cert * -cert_read(int fd) +cert_read(struct ibuf *b) { struct cert *p; - size_t i; if ((p = calloc(1, sizeof(struct cert))) == NULL) err(1, NULL); - io_simple_read(fd, &p->valid, sizeof(int)); - io_simple_read(fd, &p->ipsz, sizeof(size_t)); + io_read_buf(b, &p->expires, sizeof(p->expires)); + io_read_buf(b, &p->purpose, sizeof(p->purpose)); + io_read_buf(b, &p->talid, sizeof(p->talid)); + io_read_buf(b, &p->ipsz, sizeof(p->ipsz)); + io_read_buf(b, &p->asz, sizeof(p->asz)); + p->ips = calloc(p->ipsz, sizeof(struct cert_ip)); if (p->ips == NULL) err(1, NULL); - for (i = 0; i < p->ipsz; i++) - cert_ip_read(fd, &p->ips[i]); + io_read_buf(b, p->ips, p->ipsz * sizeof(p->ips[0])); - io_simple_read(fd, &p->asz, sizeof(size_t)); p->as = calloc(p->asz, sizeof(struct cert_as)); if (p->as == NULL) err(1, NULL); - for (i = 0; i < p->asz; i++) - cert_as_read(fd, &p->as[i]); + io_read_buf(b, p->as, p->asz * sizeof(p->as[0])); - io_str_read(fd, &p->mft); - assert(p->mft); - io_str_read(fd, &p->notify); - io_str_read(fd, &p->repo); - io_str_read(fd, &p->crl); - io_str_read(fd, &p->aia); - io_str_read(fd, &p->aki); - io_str_read(fd, &p->ski); - assert(p->ski); + io_read_str(b, &p->mft); + io_read_str(b, &p->notify); + io_read_str(b, &p->repo); + io_read_str(b, &p->crl); + io_read_str(b, &p->aia); + io_read_str(b, &p->aki); + io_read_str(b, &p->ski); + io_read_str(b, &p->pubkey); + assert(p->mft != NULL || p->purpose == CERT_PURPOSE_BGPSEC_ROUTER); + assert(p->ski); return p; } @@ -1344,6 +1307,24 @@ auth_find(struct auth_tree *auths, const return RB_FIND(auth_tree, auths, &a); } +int +auth_insert(struct auth_tree *auths, struct cert *cert, struct auth *parent) +{ + struct auth *na; + + na = malloc(sizeof(*na)); + if (na == NULL) + err(1, NULL); + + na->parent = parent; + na->cert = cert; + + if (RB_INSERT(auth_tree, auths, na) != NULL) + err(1, "auth tree corrupted"); + + return 1; +} + static inline int authcmp(struct auth *a, struct auth *b) { @@ -1351,3 +1332,80 @@ authcmp(struct auth *a, struct auth *b) } RB_GENERATE(auth_tree, auth, entry, authcmp); + +static void +insert_brk(struct brk_tree *tree, struct cert *cert, int asid) +{ + struct brk *b, *found; + + if ((b = calloc(1, sizeof(*b))) == NULL) + err(1, NULL); + + b->asid = asid; + b->expires = cert->expires; + b->talid = cert->talid; + if ((b->ski = strdup(cert->ski)) == NULL) + err(1, NULL); + if ((b->pubkey = strdup(cert->pubkey)) == NULL) + err(1, NULL); + + /* + * Check if a similar BRK already exists in the tree. If the found BRK + * expires sooner, update it to this BRK's later expiry moment. + */ + if ((found = RB_INSERT(brk_tree, tree, b)) != NULL) { + if (found->expires < b->expires) { + found->expires = b->expires; + found->talid = b->talid; + } + free(b->ski); + free(b->pubkey); + free(b); + } +} + +/* + * Add each BGPsec Router Key into the BRK tree. + */ +void +cert_insert_brks(struct brk_tree *tree, struct cert *cert) +{ + size_t i, asid; + + for (i = 0; i < cert->asz; i++) { + switch (cert->as[i].type) { + case CERT_AS_ID: + insert_brk(tree, cert, cert->as[i].id); + break; + case CERT_AS_RANGE: + for (asid = cert->as[i].range.min; + asid <= cert->as[i].range.max; asid++) + insert_brk(tree, cert, asid); + break; + default: + warnx("invalid AS identifier type"); + continue; + } + } +} + +static inline int +brkcmp(struct brk *a, struct brk *b) +{ + int rv; + + if (a->asid > b->asid) + return 1; + if (a->asid < b->asid) + return -1; + + rv = strcmp(a->ski, b->ski); + if (rv > 0) + return 1; + if (rv < 0) + return -1; + + return strcmp(a->pubkey, b->pubkey); +} + +RB_GENERATE(brk_tree, brk, entry, brkcmp); Index: usr.sbin/rpki-client/cms.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/cms.c,v retrieving revision 1.8 diff -u -p -u -r1.8 cms.c --- usr.sbin/rpki-client/cms.c 29 Jan 2021 10:13:16 -0000 1.8 +++ usr.sbin/rpki-client/cms.c 6 Nov 2021 18:09:59 -0000 @@ -35,37 +35,24 @@ * Return the eContent as a string and set "rsz" to be its length. */ unsigned char * -cms_parse_validate(X509 **xp, const char *fn, - const char *oid, size_t *rsz) +cms_parse_validate(X509 **xp, const char *fn, const unsigned char *der, + size_t derlen, const ASN1_OBJECT *oid, size_t *rsz) { const ASN1_OBJECT *obj; ASN1_OCTET_STRING **os = NULL; - BIO *bio = NULL; CMS_ContentInfo *cms; - FILE *f; - char buf[128]; - int rc = 0, sz; + int rc = 0; STACK_OF(X509) *certs = NULL; unsigned char *res = NULL; *rsz = 0; *xp = NULL; - /* - * This is usually fopen() failure, so let it pass through to - * the handler, which will in turn ignore the entity. - */ - if ((f = fopen(fn, "rb")) == NULL) { - warn("%s", fn); + /* just fail for empty buffers, the warning was printed elsewhere */ + if (der == NULL) return NULL; - } - if ((bio = BIO_new_fp(f, BIO_CLOSE)) == NULL) { - cryptowarnx("%s: BIO_new_fp", fn); - return NULL; - } - - if ((cms = d2i_CMS_bio(bio, NULL)) == NULL) { + if ((cms = d2i_CMS_ContentInfo(NULL, &der, derlen)) == NULL) { cryptowarnx("%s: RFC 6488: failed CMS parse", fn); goto out; } @@ -75,8 +62,8 @@ cms_parse_validate(X509 **xp, const char * Verify that the self-signage is correct. */ - if (!CMS_verify(cms, NULL, NULL, - NULL, NULL, CMS_NO_SIGNER_CERT_VERIFY)) { + if (!CMS_verify(cms, NULL, NULL, NULL, NULL, + CMS_NO_SIGNER_CERT_VERIFY)) { cryptowarnx("%s: RFC 6488: CMS not self-signed", fn); goto out; } @@ -84,16 +71,18 @@ cms_parse_validate(X509 **xp, const char /* RFC 6488 section 2.1.3.1: check the object's eContentType. */ obj = CMS_get0_eContentType(cms); - if ((sz = OBJ_obj2txt(buf, sizeof(buf), obj, 1)) < 0) - cryptoerrx("OBJ_obj2txt"); - - if ((size_t)sz >= sizeof(buf)) { - warnx("%s: RFC 6488 section 2.1.3.1: " - "eContentType: OID too long", fn); + if (obj == NULL) { + warnx("%s: RFC 6488 section 2.1.3.1: eContentType: " + "OID object is NULL", fn); goto out; - } else if (strcmp(buf, oid)) { + } + if (OBJ_cmp(obj, oid) != 0) { + char buf[128], obuf[128]; + + OBJ_obj2txt(buf, sizeof(buf), obj, 1); + OBJ_obj2txt(obuf, sizeof(obuf), oid, 1); warnx("%s: RFC 6488 section 2.1.3.1: eContentType: " - "unknown OID: %s, want %s", fn, buf, oid); + "unknown OID: %s, want %s", fn, buf, obuf); goto out; } @@ -133,7 +122,6 @@ cms_parse_validate(X509 **xp, const char rc = 1; out: - BIO_free_all(bio); sk_X509_free(certs); CMS_ContentInfo_free(cms); @@ -143,4 +131,62 @@ out: } return res; +} + +/* + * Wrapper around ASN1_get_object() that preserves the current start + * state and returns a more meaningful value. + * Return zero on failure, non-zero on success. + */ +int +ASN1_frame(const char *fn, size_t sz, + const unsigned char **cnt, long *cntsz, int *tag) +{ + int ret, pcls; + + ret = ASN1_get_object(cnt, cntsz, tag, &pcls, sz); + if ((ret & 0x80)) { + cryptowarnx("%s: ASN1_get_object", fn); + return 0; + } + return ASN1_object_size((ret & 0x01) ? 2 : 0, *cntsz, *tag); +} + +/* + * Check the version field in eContent. + * Returns -1 on failure, zero on success. + */ +int +cms_econtent_version(const char *fn, const unsigned char **d, size_t dsz, + long *version) +{ + ASN1_INTEGER *aint = NULL; + long plen; + int ptag, rc = -1; + + if (!ASN1_frame(fn, dsz, d, &plen, &ptag)) + goto out; + if (ptag != 0) { + warnx("%s: eContent version: expected explicit tag [0]", fn); + goto out; + } + + aint = d2i_ASN1_INTEGER(NULL, d, plen); + if (aint == NULL) { + cryptowarnx("%s: eContent version: failed d2i_ASN1_INTEGER", + fn); + goto out; + } + + *version = ASN1_INTEGER_get(aint); + if (*version < 0) { + warnx("%s: eContent version: expected positive integer, got:" + " %ld", fn, *version); + goto out; + } + + rc = 0; +out: + ASN1_INTEGER_free(aint); + return rc; } Index: usr.sbin/rpki-client/crl.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/crl.c,v retrieving revision 1.10 diff -u -p -u -r1.10 crl.c --- usr.sbin/rpki-client/crl.c 29 Jan 2021 10:13:16 -0000 1.10 +++ usr.sbin/rpki-client/crl.c 6 Nov 2021 18:10:13 -0000 @@ -29,32 +29,22 @@ #include "extern.h" X509_CRL * -crl_parse(const char *fn) +crl_parse(const char *fn, const unsigned char *der, size_t len) { int rc = 0; X509_CRL *x = NULL; - BIO *bio = NULL; - FILE *f; - if ((f = fopen(fn, "rb")) == NULL) { - warn("%s", fn); + /* just fail for empty buffers, the warning was printed elsewhere */ + if (der == NULL) return NULL; - } - - if ((bio = BIO_new_fp(f, BIO_CLOSE)) == NULL) { - if (verbose > 0) - cryptowarnx("%s: BIO_new_file", fn); - return NULL; - } - if ((x = d2i_X509_CRL_bio(bio, NULL)) == NULL) { - cryptowarnx("%s: d2i_X509_CRL_bio", fn); + if ((x = d2i_X509_CRL(NULL, &der, len)) == NULL) { + cryptowarnx("%s: d2i_X509_CRL", fn); goto out; } rc = 1; out: - BIO_free_all(bio); if (rc == 0) { X509_CRL_free(x); x = NULL; Index: usr.sbin/rpki-client/encoding.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/encoding.c,v retrieving revision 1.1 diff -u -p -u -r1.1 encoding.c --- usr.sbin/rpki-client/encoding.c 1 Apr 2021 06:43:23 -0000 1.1 +++ usr.sbin/rpki-client/encoding.c 6 Nov 2021 18:14:20 -0000 @@ -14,27 +14,91 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include + #include +#include +#include #include #include #include +#include #include #include "extern.h" /* + * Load file from disk and return the buffer and size. + */ +unsigned char * +load_file(const char *name, size_t *len) +{ + unsigned char *buf = NULL; + struct stat st; + ssize_t n; + size_t size; + int fd, saved_errno; + + *len = 0; + + if ((fd = open(name, O_RDONLY)) == -1) + return NULL; + if (fstat(fd, &st) != 0) + goto err; + if (st.st_size <= 0 || st.st_size > MAX_FILE_SIZE) { + errno = EFBIG; + goto err; + } + size = (size_t)st.st_size; + if ((buf = malloc(size)) == NULL) + goto err; + n = read(fd, buf, size); + if (n == -1) + goto err; + if ((size_t)n != size) { + errno = EIO; + goto err; + } + close(fd); + *len = size; + return buf; + +err: + saved_errno = errno; + close(fd); + free(buf); + errno = saved_errno; + return NULL; +} + +/* + * Return the size of the data blob in outlen for an inlen sized base64 buffer. + * Returns 0 on success and -1 if inlen would overflow an int. + */ +int +base64_decode_len(size_t inlen, size_t *outlen) +{ + *outlen = 0; + if (inlen >= INT_MAX - 3) + return -1; + *outlen = ((inlen + 3) / 4) * 3 + 1; + return 0; +} + +/* * Decode base64 encoded string into binary buffer returned in out. * The out buffer size is stored in outlen. * Returns 0 on success or -1 for any errors. */ int -base64_decode(const unsigned char *in, unsigned char **out, size_t *outlen) +base64_decode(const unsigned char *in, size_t inlen, + unsigned char **out, size_t *outlen) { static EVP_ENCODE_CTX *ctx; unsigned char *to; - size_t inlen; - int tolen; + size_t tolen; + int evplen; if (ctx == NULL && (ctx = EVP_ENCODE_CTX_new()) == NULL) err(1, "EVP_ENCODE_CTX_new"); @@ -42,26 +106,61 @@ base64_decode(const unsigned char *in, u *out = NULL; *outlen = 0; - inlen = strlen(in); - if (inlen >= INT_MAX - 3) + if (base64_decode_len(inlen, &tolen) == -1) return -1; - tolen = ((inlen + 3) / 4) * 3 + 1; if ((to = malloc(tolen)) == NULL) return -1; + evplen = tolen; EVP_DecodeInit(ctx); - if (EVP_DecodeUpdate(ctx, to, &tolen, in, inlen) == -1) + if (EVP_DecodeUpdate(ctx, to, &evplen, in, inlen) == -1) goto fail; - *outlen = tolen; - if (EVP_DecodeFinal(ctx, to + tolen, &tolen) == -1) + *outlen = evplen; + if (EVP_DecodeFinal(ctx, to + evplen, &evplen) == -1) goto fail; - *outlen += tolen; + *outlen += evplen; *out = to; return 0; fail: free(to); return -1; +} + +/* + * Return the size of the base64 blob in outlen for a inlen sized binary buffer. + * Returns 0 on success and -1 if inlen would overflow the calculation. + */ +int +base64_encode_len(size_t inlen, size_t *outlen) +{ + *outlen = 0; + if (inlen >= INT_MAX / 2) + return -1; + *outlen = ((inlen + 2) / 3) * 4 + 1; + return 0; +} + +/* + * Encode a binary buffer into a base64 encoded string returned in out. + * Returns 0 on success or -1 for any errors. + */ +int +base64_encode(const unsigned char *in, size_t inlen, char **out) +{ + unsigned char *to; + size_t tolen; + + *out = NULL; + + if (base64_encode_len(inlen, &tolen) == -1) + return -1; + if ((to = malloc(tolen)) == NULL) + return -1; + + EVP_EncodeBlock(to, in, inlen); + *out = to; + return 0; } /* Index: usr.sbin/rpki-client/extern.h =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v retrieving revision 1.63 diff -u -p -u -r1.63 extern.h --- usr.sbin/rpki-client/extern.h 14 Apr 2021 18:05:47 -0000 1.63 +++ usr.sbin/rpki-client/extern.h 6 Nov 2021 18:10:51 -0000 @@ -101,6 +101,12 @@ struct cert_ip { }; }; +enum cert_purpose { + CERT_PURPOSE_INVALID, + CERT_PURPOSE_CA, + CERT_PURPOSE_BGPSEC_ROUTER +}; + /* * Parsed components of a validated X509 certificate stipulated by RFC * 6847 and further (within) by RFC 3779. @@ -112,6 +118,7 @@ struct cert { size_t ipsz; /* length of "ips" */ struct cert_as *as; /* list of AS numbers and ranges */ size_t asz; /* length of "asz" */ + int talid; /* cert is covered by which TAL */ char *repo; /* CA repository (rsync:// uri) */ char *mft; /* manifest (rsync:// uri) */ char *notify; /* RRDP notify (https:// uri) */ @@ -119,8 +126,10 @@ struct cert { char *aia; /* AIA (or NULL, for trust anchor) */ char *aki; /* AKI (or NULL, for trust anchor) */ char *ski; /* SKI */ - int valid; /* validated resources */ + enum cert_purpose purpose; /* BGPSec or CA */ + char *pubkey; /* Subject Public Key Info */ X509 *x509; /* the cert */ + time_t expires; /* do not use after */ }; /* @@ -136,6 +145,7 @@ struct tal { unsigned char *pkey; /* DER-encoded public key */ size_t pkeysz; /* length of pkey */ char *descr; /* basename of tal file */ + int id; /* ID of this TAL */ }; /* @@ -183,11 +193,12 @@ struct roa { uint32_t asid; /* asID of ROA (if 0, RFC 6483 sec 4) */ struct roa_ip *ips; /* IP prefixes */ size_t ipsz; /* number of IP prefixes */ + int talid; /* ROAs are covered by which TAL */ int valid; /* validated resources */ char *aia; /* AIA */ char *aki; /* AKI */ char *ski; /* SKI */ - char *tal; /* basename of TAL for this cert */ + time_t expires; /* do not use after */ }; /* @@ -206,10 +217,11 @@ struct gbr { struct vrp { RB_ENTRY(vrp) entry; struct ip_addr addr; + int talid; /* covered by which TAL */ uint32_t asid; - char *tal; /* basename of TAL for this cert */ enum afi afi; unsigned char maxlength; + time_t expires; /* transitive expiry moment */ }; /* * Tree of VRP sorted by afi, addr, maxlength and asid @@ -218,12 +230,30 @@ RB_HEAD(vrp_tree, vrp); RB_PROTOTYPE(vrp_tree, vrp, entry, vrpcmp); /* + * A single BGPsec Router Key (including ASID) + */ +struct brk { + RB_ENTRY(brk) entry; + uint32_t asid; + int talid; /* covered by which TAL */ + char *ski; /* Subject Key Identifier */ + char *pubkey; /* Subject Public Key Info */ + time_t expires; /* transitive expiry moment */ +}; +/* + * Tree of BRK sorted by asid + */ +RB_HEAD(brk_tree, brk); +RB_PROTOTYPE(brk_tree, brk, entry, brkcmp); + +/* * A single CRL */ struct crl { RB_ENTRY(crl) entry; char *aki; X509_CRL *x509_crl; + time_t expires; /* do not use after */ }; /* * Tree of CRLs sorted by uri @@ -240,8 +270,6 @@ struct auth { RB_ENTRY(auth) entry; struct cert *cert; /* owner information */ struct auth *parent; /* pointer to parent or NULL for TA cert */ - char *tal; /* basename of TAL for this cert */ - char *fn; /* FIXME: debugging */ }; /* * Tree of auth sorted by ski @@ -249,7 +277,8 @@ struct auth { RB_HEAD(auth_tree, auth); RB_PROTOTYPE(auth_tree, auth, entry, authcmp); -struct auth *auth_find(struct auth_tree *, const char *); +struct auth *auth_find(struct auth_tree *, const char *); +int auth_insert(struct auth_tree *, struct cert *, struct auth *); /* * Resource types specified by the RPKI profiles. @@ -306,13 +335,13 @@ enum publish_type { * An entity (MFT, ROA, certificate, etc.) that needs to be downloaded * and parsed. */ -struct entity { - enum rtype type; /* type of entity (not RTYPE_EOF) */ - char *file; /* local path to file */ - int has_pkey; /* whether pkey/sz is specified */ - unsigned char *pkey; /* public key (optional) */ - size_t pkeysz; /* public key length (optional) */ - char *descr; /* tal description */ +struct entity { + enum rtype type; /* type of entity (not RTYPE_EOF) */ + char *file; /* local path to file */ + int has_data; /* whether data blob is specified */ + unsigned char *data; /* optional data blob */ + size_t datasz; /* length of optional data blob */ + int talid; /* tal identifier */ TAILQ_ENTRY(entity) entries; }; TAILQ_HEAD(entityq, entity); @@ -325,14 +354,13 @@ RB_HEAD(filepath_tree, filepath); /* * Statistics collected during run-time. */ -struct stats { +struct stats { size_t tals; /* total number of locators */ size_t mfts; /* total number of manifests */ size_t mfts_fail; /* failing syntactic parse */ size_t mfts_stale; /* stale manifests */ size_t certs; /* certificates */ - size_t certs_fail; /* failing syntactic parse */ - size_t certs_invalid; /* invalid resources */ + size_t certs_fail; /* invalid certificate */ size_t roas; /* route origin authorizations */ size_t roas_fail; /* failing syntactic parse */ size_t roas_invalid; /* invalid resources */ @@ -349,51 +377,59 @@ struct stats { size_t uniqs; /* number of unique vrps */ size_t del_files; /* number of files removed in cleanup */ size_t del_dirs; /* number of directories removed in cleanup */ - char *talnames; + size_t brks; /* number of BGPsec Router Key (BRK) certificates */ struct timeval elapsed_time; struct timeval user_time; struct timeval system_time; }; struct ibuf; +struct msgbuf; /* global variables */ extern int verbose; +extern const char *tals[]; +extern const char *taldescs[]; +extern unsigned int talrepocnt[]; +extern size_t talsz; /* Routines for RPKI entities. */ -int base64_decode(const unsigned char *, unsigned char **, - size_t *); void tal_buffer(struct ibuf *, const struct tal *); void tal_free(struct tal *); -struct tal *tal_parse(const char *, char *); -char *tal_read_file(const char *); -struct tal *tal_read(int); +struct tal *tal_parse(const char *, char *, size_t); +struct tal *tal_read(struct ibuf *); void cert_buffer(struct ibuf *, const struct cert *); void cert_free(struct cert *); -struct cert *cert_parse(X509 **, const char *); -struct cert *ta_parse(X509 **, const char *, const unsigned char *, size_t); -struct cert *cert_read(int); +struct cert *cert_parse(X509 **, const char *, const unsigned char *, + size_t); +struct cert *ta_parse(X509 **, const char *, const unsigned char *, size_t, + const unsigned char *, size_t); +struct cert *cert_read(struct ibuf *); +void cert_insert_brks(struct brk_tree *, struct cert *); void mft_buffer(struct ibuf *, const struct mft *); void mft_free(struct mft *); -struct mft *mft_parse(X509 **, const char *); +struct mft *mft_parse(X509 **, const char *, const unsigned char *, + size_t); int mft_check(const char *, struct mft *); -struct mft *mft_read(int); +struct mft *mft_read(struct ibuf *); void roa_buffer(struct ibuf *, const struct roa *); void roa_free(struct roa *); -struct roa *roa_parse(X509 **, const char *); -struct roa *roa_read(int); +struct roa *roa_parse(X509 **, const char *, const unsigned char *, + size_t); +struct roa *roa_read(struct ibuf *); void roa_insert_vrps(struct vrp_tree *, struct roa *, size_t *, size_t *); void gbr_free(struct gbr *); -struct gbr *gbr_parse(X509 **, const char *); +struct gbr *gbr_parse(X509 **, const char *, const unsigned char *, + size_t); /* crl.c */ -X509_CRL *crl_parse(const char *); +X509_CRL *crl_parse(const char *, const unsigned char *, size_t); void free_crl(struct crl *); /* Validation of our objects. */ @@ -405,13 +441,20 @@ int valid_ta(const char *, struct auth int valid_cert(const char *, struct auth_tree *, const struct cert *); int valid_roa(const char *, struct auth_tree *, struct roa *); +int valid_filename(const char *); int valid_filehash(const char *, const char *, size_t); int valid_uri(const char *, size_t, const char *); +int valid_origin(const char *, const char *); -/* Working with CMS files. */ - +/* Working with CMS. */ unsigned char *cms_parse_validate(X509 **, const char *, - const char *, size_t *); + const unsigned char *, size_t, + const ASN1_OBJECT *, size_t *); +int cms_econtent_version(const char *, const unsigned char **, + size_t, long *); +/* Helper for ASN1 parsing */ +int ASN1_frame(const char *, size_t, + const unsigned char **, long *, int *); /* Work with RFC 3779 IP addresses, prefixes, ranges. */ @@ -421,11 +464,6 @@ int ip_addr_parse(const ASN1_BIT_STRIN enum afi, const char *, struct ip_addr *); void ip_addr_print(const struct ip_addr *, enum afi, char *, size_t); -void ip_addr_buffer(struct ibuf *, const struct ip_addr *); -void ip_addr_range_buffer(struct ibuf *, - const struct ip_addr_range *); -void ip_addr_read(int, struct ip_addr *); -void ip_addr_range_read(int, struct ip_addr_range *); int ip_addr_cmp(const struct ip_addr *, const struct ip_addr *); int ip_addr_check_overlap(const struct cert_ip *, const char *, const struct cert_ip *, size_t); @@ -444,7 +482,7 @@ int as_check_covered(uint32_t, uint32_ /* Parser-specific */ void entity_free(struct entity *); -void entity_read_req(int fd, struct entity *); +void entity_read_req(struct ibuf *, struct entity *); void entityq_flush(struct entityq *, struct repo *); void proc_parser(int) __attribute__((noreturn)); @@ -464,8 +502,8 @@ void rrdp_save_state(size_t, struct rr int rrdp_handle_file(size_t, enum publish_type, char *, char *, size_t, char *, size_t); char *repo_filename(const struct repo *, const char *); -struct repo *ta_lookup(struct tal *); -struct repo *repo_lookup(const char *, const char *); +struct repo *ta_lookup(int, struct tal *); +struct repo *repo_lookup(int, const char *, const char *); int repo_queued(struct repo *, struct entity *); void repo_cleanup(struct filepath_tree *); void repo_free(void); @@ -480,6 +518,8 @@ void rrdp_fetch(size_t, const char *, struct rrdp_session *); void rrdp_http_done(size_t, enum http_result, const char *); +int repo_next_timeout(int); +void repo_check_timeout(void); /* Logging (though really used for OpenSSL errors). */ @@ -491,31 +531,45 @@ void cryptoerrx(const char *, ...) /* Encoding functions for hex and base64. */ -int base64_decode(const unsigned char *, unsigned char **, - size_t *); +unsigned char *load_file(const char *, size_t *); +int base64_decode_len(size_t, size_t *); +int base64_decode(const unsigned char *, size_t, + unsigned char **, size_t *); +int base64_encode_len(size_t, size_t *); +int base64_encode(const unsigned char *, size_t, char **); char *hex_encode(const unsigned char *, size_t); /* Functions for moving data between processes. */ -void io_socket_blocking(int); -void io_socket_nonblocking(int); +struct ibuf *io_new_buffer(void); void io_simple_buffer(struct ibuf *, const void *, size_t); void io_buf_buffer(struct ibuf *, const void *, size_t); void io_str_buffer(struct ibuf *, const char *); -void io_simple_read(int, void *, size_t); -void io_buf_read_alloc(int, void **, size_t *); -void io_str_read(int, char **); -int io_recvfd(int, void *, size_t); +void io_close_buffer(struct msgbuf *, struct ibuf *); +void io_read_buf(struct ibuf *, void *, size_t); +void io_read_str(struct ibuf *, char **); +void io_read_buf_alloc(struct ibuf *, void **, size_t *); +struct ibuf *io_buf_read(int, struct ibuf **); +struct ibuf *io_buf_recvfd(int, struct ibuf **); /* X509 helpers. */ -char *hex_encode(const unsigned char *, size_t); char *x509_get_aia(X509 *, const char *); char *x509_get_aki(X509 *, int, const char *); char *x509_get_ski(X509 *, const char *); +int x509_get_expire(X509 *, const char *, time_t *); char *x509_get_crl(X509 *, const char *); char *x509_crl_get_aki(X509_CRL *, const char *); +char *x509_get_pubkey(X509 *, const char *); +enum cert_purpose x509_get_purpose(X509 *, const char *); + +/* printers */ +void tal_print(const struct tal *); +void cert_print(const struct cert *); +void mft_print(const struct mft *); +void roa_print(const struct roa *); +void gbr_print(const struct gbr *); /* Output! */ @@ -525,21 +579,54 @@ extern int outformats; #define FORMAT_CSV 0x04 #define FORMAT_JSON 0x08 -int outputfiles(struct vrp_tree *v, struct stats *); +int outputfiles(struct vrp_tree *v, struct brk_tree *b, + struct stats *); int outputheader(FILE *, struct stats *); -int output_bgpd(FILE *, struct vrp_tree *, struct stats *); -int output_bird1v4(FILE *, struct vrp_tree *, struct stats *); -int output_bird1v6(FILE *, struct vrp_tree *, struct stats *); -int output_bird2(FILE *, struct vrp_tree *, struct stats *); -int output_csv(FILE *, struct vrp_tree *, struct stats *); -int output_json(FILE *, struct vrp_tree *, struct stats *); +int output_bgpd(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); +int output_bird1v4(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); +int output_bird1v6(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); +int output_bird2(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); +int output_csv(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); +int output_json(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); -void logx(const char *fmt, ...) +void logx(const char *fmt, ...) __attribute__((format(printf, 1, 2))); +time_t getmonotime(void); int mkpath(const char *); -#define RPKI_PATH_OUT_DIR "/var/db/rpki-client" -#define RPKI_PATH_BASE_DIR "/var/cache/rpki-client" +#define RPKI_PATH_OUT_DIR "/var/db/rpki-client" +#define RPKI_PATH_BASE_DIR "/var/cache/rpki-client" + +/* Maximum number of IP and AS ranges accepted in any single file */ +#define MAX_IP_SIZE 200000 +#define MAX_AS_SIZE 200000 + +/* Maximum acceptable URI length */ +#define MAX_URI_LENGTH 2048 + +/* Maximum acceptable file size */ +#define MAX_FILE_SIZE 2000000 + +/* Maximum number of FileAndHash entries per manifest. */ +#define MAX_MANIFEST_ENTRIES 100000 + +/* Maximum depth of the RPKI tree. */ +#define MAX_CERT_DEPTH 12 + +/* Maximum number of concurrent rsync processes. */ +#define MAX_RSYNC_PROCESSES 16 + +/* Maximum allowd repositories per tal */ +#define MAX_REPO_PER_TAL 1000 + +/* Timeout for repository synchronisation, in seconds */ +#define MAX_REPO_TIMEOUT (15 * 60) #endif /* ! EXTERN_H */ Index: usr.sbin/rpki-client/gbr.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/gbr.c,v retrieving revision 1.9 diff -u -p -u -r1.9 gbr.c --- usr.sbin/rpki-client/gbr.c 29 Mar 2021 06:50:44 -0000 1.9 +++ usr.sbin/rpki-client/gbr.c 6 Nov 2021 18:11:02 -0000 @@ -36,13 +36,15 @@ struct parse { struct gbr *res; /* results */ }; +static ASN1_OBJECT *gbr_oid; + /* * Parse a full RFC 6493 file and signed by the certificate "cacert" * (the latter is optional and may be passed as NULL to disable). * Returns the payload or NULL if the document was malformed. */ struct gbr * -gbr_parse(X509 **x509, const char *fn) +gbr_parse(X509 **x509, const char *fn, const unsigned char *der, size_t len) { struct parse p; size_t cmsz; @@ -52,9 +54,14 @@ gbr_parse(X509 **x509, const char *fn) p.fn = fn; /* OID from section 9.1, RFC 6493. */ + if (gbr_oid == NULL) { + gbr_oid = OBJ_txt2obj("1.2.840.113549.1.9.16.1.35", 1); + if (gbr_oid == NULL) + errx(1, "OBJ_txt2obj for %s failed", + "1.2.840.113549.1.9.16.1.35"); + } - cms = cms_parse_validate(x509, fn, - "1.2.840.113549.1.9.16.1.35", &cmsz); + cms = cms_parse_validate(x509, fn, der, len, gbr_oid, &cmsz); if (cms == NULL) return NULL; Index: usr.sbin/rpki-client/http.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/http.c,v retrieving revision 1.30 diff -u -p -u -r1.30 http.c --- usr.sbin/rpki-client/http.c 15 Apr 2021 16:07:21 -0000 1.30 +++ usr.sbin/rpki-client/http.c 6 Nov 2021 18:17:04 -0000 @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -66,78 +67,142 @@ #include "extern.h" -#define HTTP_USER_AGENT "OpenBSD rpki-client" -#define HTTP_BUF_SIZE (32 * 1024) -#define MAX_CONNECTIONS 12 - -#define WANT_POLLIN 1 -#define WANT_POLLOUT 2 +#define HTTP_USER_AGENT "OpenBSD rpki-client" +#define HTTP_BUF_SIZE (32 * 1024) +#define HTTP_IDLE_TIMEOUT 10 +#define HTTP_IO_TIMEOUT (3 * 60) +#define MAX_CONNECTIONS 64 +#define MAX_CONTENTLEN (2 * 1024 * 1024 * 1024LL) +#define NPFDS (MAX_CONNECTIONS + 1) + +enum res { + DONE, + WANT_POLLIN, + WANT_POLLOUT, +}; enum http_state { STATE_FREE, - STATE_INIT, STATE_CONNECT, STATE_TLSCONNECT, + STATE_PROXY_REQUEST, + STATE_PROXY_STATUS, + STATE_PROXY_RESPONSE, STATE_REQUEST, STATE_RESPONSE_STATUS, STATE_RESPONSE_HEADER, STATE_RESPONSE_DATA, - STATE_RESPONSE_CHUNKED, + STATE_RESPONSE_CHUNKED_HEADER, + STATE_RESPONSE_CHUNKED_TRAILER, STATE_WRITE_DATA, - STATE_DONE, + STATE_IDLE, + STATE_CLOSE, }; struct http_proxy { char *proxyhost; - char *proxyuser; - char *proxypw; -}; + char *proxyport; + char *proxyauth; +} proxy; struct http_connection { - char *url; + LIST_ENTRY(http_connection) entry; char *host; char *port; - const char *path; /* points into url */ - char *modified_since; char *last_modified; + char *redir_uri; + struct http_request *req; + struct pollfd *pfd; struct addrinfo *res0; struct addrinfo *res; struct tls *tls; char *buf; size_t bufsz; size_t bufpos; - size_t id; off_t iosz; + off_t totalsz; + time_t idle_time; + time_t io_time; int status; - int redirect_loop; int fd; - int outfd; + int chunked; + int keep_alive; short events; - short chunked; enum http_state state; }; -struct msgbuf msgq; -struct sockaddr_storage http_bindaddr; -struct tls_config *tls_config; -uint8_t *tls_ca_mem; -size_t tls_ca_size; +LIST_HEAD(http_conn_list, http_connection); + +struct http_request { + TAILQ_ENTRY(http_request) entry; + char *uri; + char *modified_since; + char *host; + char *port; + const char *path; /* points into uri */ + size_t id; + int outfd; + int redirect_loop; +}; + +TAILQ_HEAD(http_req_queue, http_request); +static struct http_conn_list active = LIST_HEAD_INITIALIZER(active); +static struct http_conn_list idle = LIST_HEAD_INITIALIZER(idle); +static struct http_req_queue queue = TAILQ_HEAD_INITIALIZER(queue); +static size_t http_conn_count; + +static struct msgbuf msgq; +static struct sockaddr_storage http_bindaddr; +static struct tls_config *tls_config; +static uint8_t *tls_ca_mem; +static size_t tls_ca_size; + +/* HTTP request API */ +static void http_req_new(size_t, char *, char *, int, int); +static void http_req_free(struct http_request *); +static void http_req_done(size_t, enum http_result, const char *); +static void http_req_fail(size_t); +static int http_req_schedule(struct http_request *); + +/* HTTP connection API */ +static void http_new(struct http_request *); static void http_free(struct http_connection *); -static int http_tls_handshake(struct http_connection *); -static int http_write(struct http_connection *); +static enum res http_done(struct http_connection *, enum http_result); +static enum res http_failed(struct http_connection *); + +/* HTTP connection FSM functions */ +static void http_do(struct http_connection *, + enum res (*)(struct http_connection *)); + +/* These functions can be used with http_do() */ +static enum res http_connect(struct http_connection *); +static enum res http_request(struct http_connection *); +static enum res http_close(struct http_connection *); +static enum res http_handle(struct http_connection *); + +/* Internal state functions used by the above functions */ +static enum res http_finish_connect(struct http_connection *); +static enum res proxy_connect(struct http_connection *); +static enum res http_tls_connect(struct http_connection *); +static enum res http_tls_handshake(struct http_connection *); +static enum res http_read(struct http_connection *); +static enum res http_write(struct http_connection *); +static enum res proxy_read(struct http_connection *); +static enum res proxy_write(struct http_connection *); +static enum res data_write(struct http_connection *); /* * Return a string that can be used in error message to identify the * connection. */ static const char * -http_info(const char *url) +http_info(const char *uri) { static char buf[80]; - if (strnvis(buf, url, sizeof buf, VIS_SAFE) >= (int)sizeof buf) { + if (strnvis(buf, uri, sizeof buf, VIS_SAFE) >= (int)sizeof buf) { /* overflow, add indicator */ memcpy(buf + sizeof buf - 4, "...", 4); } @@ -212,6 +277,144 @@ url_encode(const char *path) return (epath); } +static char +hextochar(const char *str) +{ + unsigned char c, ret; + + c = str[0]; + ret = c; + if (isalpha(c)) + ret -= isupper(c) ? 'A' - 10 : 'a' - 10; + else + ret -= '0'; + ret *= 16; + + c = str[1]; + ret += c; + if (isalpha(c)) + ret -= isupper(c) ? 'A' - 10 : 'a' - 10; + else + ret -= '0'; + return ret; +} + +static char * +url_decode(const char *str) +{ + char *ret, c; + int i, reallen; + + if (str == NULL) + return NULL; + if ((ret = malloc(strlen(str) + 1)) == NULL) + err(1, "Can't allocate memory for URL decoding"); + for (i = 0, reallen = 0; str[i] != '\0'; i++, reallen++, ret++) { + c = str[i]; + if (c == '+') { + *ret = ' '; + continue; + } + /* + * Cannot use strtol here because next char + * after %xx may be a digit. + */ + if (c == '%' && isxdigit((unsigned char)str[i + 1]) && + isxdigit((unsigned char)str[i + 2])) { + *ret = hextochar(&str[i + 1]); + i += 2; + continue; + } + *ret = c; + } + *ret = '\0'; + return ret - reallen; +} + +static char * +recode_credentials(const char *userinfo) +{ + char *ui, *creds; + size_t ulen; + + /* url-decode the user and pass */ + ui = url_decode(userinfo); + + ulen = strlen(ui); + if (base64_encode(ui, ulen, &creds) == -1) + errx(1, "error in base64 encoding"); + free(ui); + return (creds); +} + +/* + * Parse a proxy URI and split it up into host, port and userinfo. + */ +static void +proxy_parse_uri(char *uri) +{ + char *host, *port = NULL, *cred, *cookie = NULL; + + if (uri == NULL) + return; + + if (strncasecmp(uri, "http://", 7) != 0) + errx(1, "%s: http_proxy not using http schema", http_info(uri)); + + host = uri + 7; + if ((host = strndup(host, strcspn(host, "/"))) == NULL) + err(1, NULL); + + cred = host; + host = strchr(cred, '@'); + if (host != NULL) + *host++ = '\0'; + else { + host = cred; + cred = NULL; + } + + if (*host == '[') { + char *hosttail; + + if ((hosttail = strrchr(host, ']')) == NULL) + errx(1, "%s: unmatched opening bracket", + http_info(uri)); + if (hosttail[1] == '\0' || hosttail[1] == ':') + host++; + if (hosttail[1] == ':') + port = hosttail + 2; + *hosttail = '\0'; + } else { + if ((port = strrchr(host, ':')) != NULL) + *port++ = '\0'; + } + + if (port == NULL) + port = "443"; + + if (cred != NULL) { + if (strchr(cred, ':') == NULL) + errx(1, "%s: malformed proxy url", http_info(uri)); + cred = recode_credentials(cred); + if (asprintf(&cookie, "Proxy-Authorization: Basic %s\r\n", + cred) == -1) + err(1, NULL); + free(cred); + } else + if ((cookie = strdup("")) == NULL) + err(1, NULL); + + proxy.proxyhost = host; + proxy.proxyport = port; + proxy.proxyauth = cookie; +} + +/* + * Parse a URI and split it up into host, port and path. + * Does some basic URI validation. Both host and port need to be freed + * by the caller whereas path points into the uri. + */ static int http_parse_uri(char *uri, char **ohost, char **oport, char **opath) { @@ -231,8 +434,13 @@ http_parse_uri(char *uri, char **ohost, warnx("%s: preposterous host length", http_info(uri)); return -1; } + + if (memchr(host, '@', path - host) != NULL) { + warnx("%s: URI with userinfo not supported", http_info(uri)); + return -1; + } + if (*host == '[') { - char *scope; if ((hosttail = memrchr(host, ']', path - host)) == NULL) { warnx("%s: unmatched opening bracket", http_info(uri)); return -1; @@ -241,8 +449,6 @@ http_parse_uri(char *uri, char **ohost, host++; if (hosttail[1] == ':') port = hosttail + 2; - if ((scope = memchr(host, '%', hosttail - host)) != NULL) - hosttail = scope; } else { if ((hosttail = memrchr(host, ':', path - host)) != NULL) port = hosttail + 1; @@ -269,8 +475,12 @@ http_parse_uri(char *uri, char **ohost, return 0; } +/* + * Lookup the IP addresses for host:port. + * Returns 0 on success and -1 on failure. + */ static int -http_resolv(struct http_connection *conn, const char *host, const char *port) +http_resolv(struct addrinfo **res, const char *host, const char *port) { struct addrinfo hints; int error; @@ -278,13 +488,13 @@ http_resolv(struct http_connection *conn memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; - error = getaddrinfo(host, port, &hints, &conn->res0); + error = getaddrinfo(host, port, &hints, res); /* * If the services file is corrupt/missing, fall back * on our hard-coded defines. */ if (error == EAI_SERVICE) - error = getaddrinfo(host, "443", &hints, &conn->res0); + error = getaddrinfo(host, "443", &hints, res); if (error != 0) { warnx("%s: %s", host, gai_strerror(error)); return -1; @@ -293,80 +503,190 @@ http_resolv(struct http_connection *conn return 0; } +/* + * Create and queue a new request. + */ static void -http_done(size_t id, enum http_result res, const char *last_modified) +http_req_new(size_t id, char *uri, char *modified_since, int count, int outfd) { - struct ibuf *b; + struct http_request *req; + char *host, *port, *path; + + if (http_parse_uri(uri, &host, &port, &path) == -1) { + free(uri); + free(modified_since); + close(outfd); + http_req_fail(id); + return; + } - if ((b = ibuf_dynamic(64, UINT_MAX)) == NULL) + if ((req = calloc(1, sizeof(*req))) == NULL) err(1, NULL); + + req->id = id; + req->outfd = outfd; + req->host = host; + req->port = port; + req->path = path; + req->uri = uri; + req->modified_since = modified_since; + req->redirect_loop = count; + + TAILQ_INSERT_TAIL(&queue, req, entry); +} + +/* + * Free a request, request is not allowed to be on the req queue. + */ +static void +http_req_free(struct http_request *req) +{ + if (req == NULL) + return; + + free(req->host); + free(req->port); + /* no need to free req->path it points into req->uri */ + free(req->uri); + free(req->modified_since); + + if (req->outfd != -1) + close(req->outfd); +} + +/* + * Enqueue request response + */ +static void +http_req_done(size_t id, enum http_result res, const char *last_modified) +{ + struct ibuf *b; + + b = io_new_buffer(); io_simple_buffer(b, &id, sizeof(id)); io_simple_buffer(b, &res, sizeof(res)); io_str_buffer(b, last_modified); - ibuf_close(&msgq, b); + io_close_buffer(&msgq, b); } +/* + * Enqueue request failure response + */ static void -http_fail(size_t id) +http_req_fail(size_t id) { struct ibuf *b; enum http_result res = HTTP_FAILED; - if ((b = ibuf_dynamic(8, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &id, sizeof(id)); io_simple_buffer(b, &res, sizeof(res)); io_str_buffer(b, NULL); - ibuf_close(&msgq, b); + io_close_buffer(&msgq, b); } -static struct http_connection * -http_new(size_t id, char *uri, char *modified_since, int outfd) +/* + * Schedule new requests until maximum number of connections is reached. + * Try to reuse an idle connection if one exists that matches host and port. + */ +static int +http_req_schedule(struct http_request *req) { struct http_connection *conn; - char *host, *port, *path; - if (http_parse_uri(uri, &host, &port, &path) == -1) { - free(uri); - free(modified_since); - close(outfd); - http_fail(id); - return NULL; + TAILQ_REMOVE(&queue, req, entry); + + /* check list of idle connections first */ + LIST_FOREACH(conn, &idle, entry) { + if (strcmp(conn->host, req->host) != 0) + continue; + if (strcmp(conn->port, req->port) != 0) + continue; + + LIST_REMOVE(conn, entry); + LIST_INSERT_HEAD(&active, conn, entry); + + /* use established connection */ + conn->req = req; + conn->idle_time = 0; + + /* start request */ + http_do(conn, http_request); + if (conn->state == STATE_FREE) + http_free(conn); + return 1; } + if (http_conn_count < MAX_CONNECTIONS) { + http_new(req); + return 1; + } + + /* no more slots free, requeue */ + TAILQ_INSERT_HEAD(&queue, req, entry); + return 0; +} + +/* + * Create a new HTTP connection which will be used for the HTTP request req. + * On errors a req faulure is issued and both connection and request are freed. + */ +static void +http_new(struct http_request *req) +{ + struct http_connection *conn; + if ((conn = calloc(1, sizeof(*conn))) == NULL) err(1, NULL); - conn->id = id; conn->fd = -1; - conn->outfd = outfd; - conn->host = host; - conn->port = port; - conn->path = path; - conn->url = uri; - conn->modified_since = modified_since; - conn->state = STATE_INIT; + conn->req = req; + if ((conn->host = strdup(req->host)) == NULL) + err(1, NULL); + if ((conn->port = strdup(req->port)) == NULL) + err(1, NULL); - /* TODO proxy support (overload of host and port) */ + LIST_INSERT_HEAD(&active, conn, entry); + http_conn_count++; - if (http_resolv(conn, host, port) == -1) { - http_fail(conn->id); - http_free(conn); - return NULL; + if (proxy.proxyhost != NULL) { + if (http_resolv(&conn->res0, proxy.proxyhost, + proxy.proxyport) == -1) { + http_req_fail(req->id); + http_free(conn); + return; + } + } else { + if (http_resolv(&conn->res0, conn->host, conn->port) == -1) { + http_req_fail(req->id); + http_free(conn); + return; + } } - return conn; + /* connect and start request */ + http_do(conn, http_connect); + if (conn->state == STATE_FREE) + http_free(conn); } +/* + * Free a no longer active connection, releasing all memory and closing + * any open file descriptor. + */ static void http_free(struct http_connection *conn) { - free(conn->url); + assert(conn->state == STATE_FREE); + + LIST_REMOVE(conn, entry); + http_conn_count--; + + http_req_free(conn->req); free(conn->host); free(conn->port); - /* no need to free conn->path it points into conn->url */ - free(conn->modified_since); free(conn->last_modified); + free(conn->redir_uri); free(conn->buf); if (conn->res0 != NULL) @@ -376,39 +696,113 @@ http_free(struct http_connection *conn) if (conn->fd != -1) close(conn->fd); - close(conn->outfd); free(conn); } +/* + * Called when a request on this connection is finished. + * Move connection into idle state and onto idle queue. + * If there is a request connected to it send back a response + * with http_result res, else ignore the res. + */ +static enum res +http_done(struct http_connection *conn, enum http_result res) +{ + assert(conn->bufpos == 0); + assert(conn->iosz == 0); + assert(conn->chunked == 0); + assert(conn->redir_uri == NULL); -static int + conn->state = STATE_IDLE; + conn->idle_time = getmonotime() + HTTP_IDLE_TIMEOUT; + + if (conn->req) { + http_req_done(conn->req->id, res, conn->last_modified); + http_req_free(conn->req); + conn->req = NULL; + } + + if (!conn->keep_alive) + return http_close(conn); + + LIST_REMOVE(conn, entry); + LIST_INSERT_HEAD(&idle, conn, entry); + + /* reset status and keep-alive for good measures */ + conn->status = 0; + conn->keep_alive = 0; + + return WANT_POLLIN; +} + +/* + * Called in case of error, moves connection into free state. + * This will skip proper shutdown of the TLS session. + * If a request is pending fail and free the request. + */ +static enum res +http_failed(struct http_connection *conn) +{ + conn->state = STATE_FREE; + + if (conn->req) { + http_req_fail(conn->req->id); + http_req_free(conn->req); + conn->req = NULL; + } + + return DONE; +} + +/* + * Call the function f and update the connection events based + * on the return value. + */ +static void +http_do(struct http_connection *conn, enum res (*f)(struct http_connection *)) +{ + switch (f(conn)) { + case DONE: + conn->events = 0; + break; + case WANT_POLLIN: + conn->events = POLLIN; + break; + case WANT_POLLOUT: + conn->events = POLLOUT; + break; + default: + errx(1, "%s: unexpected function return", + http_info(conn->host)); + } +} + +/* + * Connection successfully establish, initiate TLS handshake or proxy request. + */ +static enum res http_connect_done(struct http_connection *conn) { freeaddrinfo(conn->res0); conn->res0 = NULL; conn->res = NULL; -#if 0 - /* TODO proxy connect */ - if (proxyenv) - proxy_connect(conn->fd, sslhost, proxy_credentials); */ -#endif - - return 0; + if (proxy.proxyhost != NULL) + return proxy_connect(conn); + return http_tls_connect(conn); } -static int +/* + * Start an asynchronous connect. + */ +static enum res http_connect(struct http_connection *conn) { const char *cause = NULL; + assert(conn->fd == -1); conn->state = STATE_CONNECT; - if (conn->fd != -1) { - close(conn->fd); - conn->fd = -1; - } - /* start the loop below with first or next address */ if (conn->res == NULL) conn->res = conn->res0; @@ -457,17 +851,17 @@ http_connect(struct http_connection *con if (conn->fd == -1) { if (cause != NULL) - warn("%s: %s", http_info(conn->url), cause); - freeaddrinfo(conn->res0); - conn->res0 = NULL; - conn->res = NULL; - return -1; + warn("%s: %s", http_info(conn->req->uri), cause); + return http_failed(conn); } return http_connect_done(conn); } -static int +/* + * Called once an asynchronus connect request finished. + */ +static enum res http_finish_connect(struct http_connection *conn) { int error = 0; @@ -475,69 +869,122 @@ http_finish_connect(struct http_connecti len = sizeof(error); if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &error, &len) == -1) { - warn("%s: getsockopt SO_ERROR", http_info(conn->url)); - /* connection will be closed by http_connect() */ - return -1; + warn("%s: getsockopt SO_ERROR", http_info(conn->req->uri)); + goto fail; } if (error != 0) { errno = error; - warn("%s: connect", http_info(conn->url)); - return -1; + warn("%s: connect", http_info(conn->req->uri)); + goto fail; } return http_connect_done(conn); + +fail: + close(conn->fd); + conn->fd = -1; + + return http_connect(conn); } -static int +/* + * Initiate TLS session on a new connection. + */ +static enum res http_tls_connect(struct http_connection *conn) { + assert(conn->state == STATE_CONNECT); + conn->state = STATE_TLSCONNECT; + if ((conn->tls = tls_client()) == NULL) { warn("tls_client"); - return -1; + return http_failed(conn); } if (tls_configure(conn->tls, tls_config) == -1) { - warnx("%s: TLS configuration: %s\n", http_info(conn->url), + warnx("%s: TLS configuration: %s\n", http_info(conn->req->uri), tls_error(conn->tls)); - return -1; + return http_failed(conn); } if (tls_connect_socket(conn->tls, conn->fd, conn->host) == -1) { - warnx("%s: TLS connect: %s\n", http_info(conn->url), + warnx("%s: TLS connect: %s\n", http_info(conn->req->uri), tls_error(conn->tls)); - return -1; + return http_failed(conn); } + return http_tls_handshake(conn); } -static int +/* + * Do the tls_handshake and then send out the HTTP request. + */ +static enum res http_tls_handshake(struct http_connection *conn) { switch (tls_handshake(conn->tls)) { - case 0: - return 0; + case -1: + warnx("%s: TLS handshake: %s", http_info(conn->req->uri), + tls_error(conn->tls)); + return http_failed(conn); case TLS_WANT_POLLIN: return WANT_POLLIN; case TLS_WANT_POLLOUT: return WANT_POLLOUT; } - warnx("%s: TLS handshake: %s", http_info(conn->url), - tls_error(conn->tls)); - return -1; + + return http_request(conn); } -static int +static enum res +proxy_connect(struct http_connection *conn) +{ + char *host; + int r; + + assert(conn->state == STATE_CONNECT); + conn->state = STATE_PROXY_REQUEST; + + /* Construct the Host header from host and port info */ + if (strchr(conn->host, ':')) { + if (asprintf(&host, "[%s]:%s", conn->host, conn->port) == -1) + err(1, NULL); + + } else { + if (asprintf(&host, "%s:%s", conn->host, conn->port) == -1) + err(1, NULL); + } + + free(conn->buf); + conn->bufpos = 0; + /* XXX handle auth */ + if ((r = asprintf(&conn->buf, "CONNECT %s HTTP/1.1\r\n" + "User-Agent: " HTTP_USER_AGENT "\r\n%s\r\n", host, + proxy.proxyauth)) == -1) + err(1, NULL); + conn->bufsz = r; + + free(host); + + return proxy_write(conn); +} + +/* + * Build the HTTP request and send it out. + */ +static enum res http_request(struct http_connection *conn) { char *host, *epath, *modified_since; int r, with_port = 0; - /* TODO adjust request for HTTP proxy setups */ + assert(conn->state == STATE_IDLE || conn->state == STATE_TLSCONNECT); + conn->state = STATE_REQUEST; /* * Send port number only if it's specified and does not equal * the default. Some broken HTTP servers get confused if you explicitly * send them the port number. */ - if (conn->port && strcmp(conn->port, "443") != 0) + if (strcmp(conn->port, "443") != 0) with_port = 1; /* Construct the Host header from host and port info */ @@ -555,12 +1002,12 @@ http_request(struct http_connection *con /* * Construct and send the request. Proxy requests don't want leading /. */ - epath = url_encode(conn->path); + epath = url_encode(conn->req->path); modified_since = NULL; - if (conn->modified_since) { + if (conn->req->modified_since != NULL) { if (asprintf(&modified_since, "If-Modified-Since: %s\r\n", - conn->modified_since) == -1) + conn->req->modified_since) == -1) err(1, NULL); } @@ -568,9 +1015,10 @@ http_request(struct http_connection *con conn->bufpos = 0; if ((r = asprintf(&conn->buf, "GET /%s HTTP/1.1\r\n" - "Connection: close\r\n" + "Host: %s\r\n" + "Accept-Encoding: identity\r\n" "User-Agent: " HTTP_USER_AGENT "\r\n" - "Host: %s\r\n%s\r\n", + "%s\r\n", epath, host, modified_since ? modified_since : "")) == -1) err(1, NULL); @@ -580,57 +1028,79 @@ http_request(struct http_connection *con free(host); free(modified_since); - return WANT_POLLOUT; + return http_write(conn); } +/* + * Parse the HTTP status line. + * Return 0 for status codes 100, 103, 200, 203, 301-304, 307-308. + * The other 1xx and 2xx status codes are explicitly not handled and are + * considered an error. + * Failure codes and other errors return -1. + * The redirect loop limit is enforced here. + */ static int http_parse_status(struct http_connection *conn, char *buf) { +#define HTTP_11 "HTTP/1.1 " const char *errstr; char *cp, ststr[4]; char gerror[200]; int status; + /* Check if the protocol is 1.1 and enable keep-alive in that case */ + if (strncmp(buf, HTTP_11, strlen(HTTP_11)) == 0) + conn->keep_alive = 1; + cp = strchr(buf, ' '); if (cp == NULL) { - warnx("Improper response from %s", http_info(conn->url)); + warnx("Improper response from %s", http_info(conn->host)); return -1; } else cp++; strlcpy(ststr, cp, sizeof(ststr)); - status = strtonum(ststr, 200, 599, &errstr); + status = strtonum(ststr, 100, 599, &errstr); if (errstr != NULL) { strnvis(gerror, cp, sizeof gerror, VIS_SAFE); - warnx("Error retrieving %s: %s", http_info(conn->url), gerror); + warnx("Error retrieving %s: %s", http_info(conn->host), + gerror); return -1; } switch (status) { - case 301: - case 302: - case 303: - case 307: - case 308: - if (conn->redirect_loop++ > 10) { + case 301: /* Redirect: moved permanently */ + case 302: /* Redirect: found / moved temporarily */ + case 303: /* Redirect: see other */ + case 307: /* Redirect: temporary redirect */ + case 308: /* Redirect: permanent redirect */ + if (conn->req->redirect_loop++ > 10) { warnx("%s: Too many redirections requested", - http_info(conn->url)); + http_info(conn->host)); return -1; } /* FALLTHROUGH */ - case 200: - case 304: + case 100: /* Informational: continue (ignored) */ + case 103: /* Informational: early hints (ignored) */ + /* FALLTHROUGH */ + case 200: /* Success: OK */ + case 203: /* Success: non-authoritative information (proxy) */ + case 304: /* Redirect: not modified */ conn->status = status; break; default: strnvis(gerror, cp, sizeof gerror, VIS_SAFE); - warnx("Error retrieving %s: %s", http_info(conn->url), gerror); - break; + warnx("Error retrieving %s: %s", http_info(conn->host), + gerror); + return -1; } return 0; } +/* + * Returns true if the connection status is any of the redirect codes. + */ static inline int http_isredirect(struct http_connection *conn) { @@ -640,44 +1110,38 @@ http_isredirect(struct http_connection * return 0; } -static int -http_redirect(struct http_connection *conn, char *uri) +static inline int +http_isok(struct http_connection *conn) { - char *host, *port, *path; + if (conn->status >= 200 && conn->status < 300) + return 1; + return 0; +} - logx("redirect to %s", http_info(uri)); +static void +http_redirect(struct http_connection *conn) +{ + char *uri, *mod_since = NULL; + int outfd; - if (http_parse_uri(uri, &host, &port, &path) == -1) { - free(uri); - return -1; - } + /* move uri and fd out for new request */ + outfd = conn->req->outfd; + conn->req->outfd = -1; - free(conn->url); - conn->url = uri; - free(conn->host); - conn->host = host; - free(conn->port); - conn->port = port; - conn->path = path; - /* keep modified_since since that is part of the request */ - free(conn->last_modified); - conn->last_modified = NULL; - free(conn->buf); - conn->buf = NULL; - conn->bufpos = 0; - conn->bufsz = 0; - tls_close(conn->tls); - tls_free(conn->tls); - conn->tls = NULL; - close(conn->fd); - conn->state = STATE_INIT; + uri = conn->redir_uri; + conn->redir_uri = NULL; - /* TODO proxy support (overload of host and port) */ + if (conn->req->modified_since) + if ((mod_since = strdup(conn->req->modified_since)) == NULL) + err(1, NULL); - if (http_resolv(conn, host, port) == -1) - return -1; + logx("redirect to %s", http_info(uri)); + http_req_new(conn->req->id, uri, mod_since, conn->req->redirect_loop, + outfd); - return -2; + /* clear request before moving connection to idle */ + http_req_free(conn->req); + conn->req = NULL; } static int @@ -685,6 +1149,7 @@ http_parse_header(struct http_connection { #define CONTENTLEN "Content-Length: " #define LOCATION "Location: " +#define CONNECTION "Connection: " #define TRANSFER_ENCODING "Transfer-Encoding: " #define LAST_MODIFIED "Last-Modified: " const char *errstr; @@ -700,10 +1165,10 @@ http_parse_header(struct http_connection cp += sizeof(CONTENTLEN) - 1; if ((s = strcspn(cp, " \t")) != 0) *(cp+s) = 0; - conn->iosz = strtonum(cp, 0, LLONG_MAX, &errstr); + conn->iosz = strtonum(cp, 0, MAX_CONTENTLEN, &errstr); if (errstr != NULL) { warnx("Content-Length of %s is %s", - http_info(conn->url), errstr); + http_info(conn->req->uri), errstr); return -1; } } else if (http_isredirect(conn) && @@ -719,7 +1184,7 @@ http_parse_header(struct http_connection locbase = NULL; cp++; } else { - locbase = strdup(conn->path); + locbase = strdup(conn->req->path); if (locbase == NULL) err(1, NULL); loctail = strchr(locbase, '#'); @@ -737,9 +1202,8 @@ http_parse_header(struct http_connection } /* Construct URL from relative redirect */ if (asprintf(&redirurl, "%.*s/%s%s", - (int)(conn->path - conn->url), conn->url, - locbase ? locbase : "", - cp) == -1) + (int)(conn->req->path - conn->req->uri), + conn->req->uri, locbase ? locbase : "", cp) == -1) err(1, "Cannot build redirect URL"); free(locbase); } else if ((redirurl = strdup(cp)) == NULL) @@ -747,13 +1211,20 @@ http_parse_header(struct http_connection loctail = strchr(redirurl, '#'); if (loctail != NULL) *loctail = '\0'; - return http_redirect(conn, redirurl); + conn->redir_uri = redirurl; } else if (strncasecmp(cp, TRANSFER_ENCODING, sizeof(TRANSFER_ENCODING) - 1) == 0) { cp += sizeof(TRANSFER_ENCODING) - 1; cp[strcspn(cp, " \t")] = '\0'; if (strcasecmp(cp, "chunked") == 0) conn->chunked = 1; + } else if (strncasecmp(cp, CONNECTION, sizeof(CONNECTION) - 1) == 0) { + cp += sizeof(CONNECTION) - 1; + cp[strcspn(cp, " \t")] = '\0'; + if (strcasecmp(cp, "close") == 0) + conn->keep_alive = 0; + else if (strcasecmp(cp, "keep-alive") == 0) + conn->keep_alive = 1; } else if (strncasecmp(cp, LAST_MODIFIED, sizeof(LAST_MODIFIED) - 1) == 0) { cp += sizeof(LAST_MODIFIED) - 1; @@ -764,6 +1235,12 @@ http_parse_header(struct http_connection return 1; } +/* + * Return one line from the HTTP response. + * The line returned has any possible '\r' and '\n' at the end stripped. + * The buffer is advanced to the start of the next line. + * If there is currently no full line in the buffer NULL is returned. + */ static char * http_get_line(struct http_connection *conn) { @@ -788,6 +1265,12 @@ http_get_line(struct http_connection *co return line; } +/* + * Parse the header between data chunks during chunked transfers. + * Returns 0 if a new chunk size could be correctly read. + * Returns 1 for the empty trailer lines. + * If the chuck size could not be converted properly -1 is returned. + */ static int http_parse_chunked(struct http_connection *conn, char *buf) { @@ -795,7 +1278,7 @@ http_parse_chunked(struct http_connectio char *end; unsigned long chunksize; - /* ignore empty lines, used between chunk and next header */ + /* empty lines are used as trailer */ if (*header == '\0') return 1; @@ -804,58 +1287,84 @@ http_parse_chunked(struct http_connectio errno = 0; chunksize = strtoul(header, &end, 16); if (header[0] == '\0' || *end != '\0' || (errno == ERANGE && - chunksize == ULONG_MAX) || chunksize > INT_MAX) { - warnx("%s: Invalid chunk size", http_info(conn->url)); + chunksize == ULONG_MAX) || chunksize > INT_MAX) return -1; - } - conn->iosz = chunksize; - - if (conn->iosz == 0) { - http_done(conn->id, HTTP_OK, conn->last_modified); - conn->state = STATE_DONE; - return 0; - } - return 1; + conn->iosz = chunksize; + return 0; } -static int +static enum res http_read(struct http_connection *conn) { ssize_t s; char *buf; int done; + if (conn->bufpos > 0) + goto again; + read_more: s = tls_read(conn->tls, conn->buf + conn->bufpos, conn->bufsz - conn->bufpos); if (s == -1) { - warn("%s: TLS read: %s", http_info(conn->url), + warnx("%s: TLS read: %s", http_info(conn->host), tls_error(conn->tls)); - return -1; + return http_failed(conn); } else if (s == TLS_WANT_POLLIN) { return WANT_POLLIN; } else if (s == TLS_WANT_POLLOUT) { return WANT_POLLOUT; } - if (s == 0 && conn->bufpos == 0) { - warnx("%s: short read, connection closed", - http_info(conn->url)); - return -1; + if (s == 0) { + if (conn->req) + warnx("%s: short read, connection closed", + http_info(conn->req->uri)); + return http_failed(conn); } conn->bufpos += s; again: switch (conn->state) { + case STATE_PROXY_STATUS: + buf = http_get_line(conn); + if (buf == NULL) + goto read_more; + if (http_parse_status(conn, buf) == -1) { + free(buf); + return http_failed(conn); + } + free(buf); + conn->state = STATE_PROXY_RESPONSE; + goto again; + case STATE_PROXY_RESPONSE: + while (1) { + buf = http_get_line(conn); + if (buf == NULL) + goto read_more; + /* empty line, end of header */ + if (*buf == '\0') { + free(buf); + break; + } + free(buf); + } + /* proxy is ready to take connection */ + if (conn->status == 200) { + conn->state = STATE_CONNECT; + return http_tls_connect(conn); + } + return http_failed(conn); case STATE_RESPONSE_STATUS: buf = http_get_line(conn); if (buf == NULL) goto read_more; + if (http_parse_status(conn, buf) == -1) { free(buf); - return -1; + return http_failed(conn); } free(buf); conn->state = STATE_RESPONSE_HEADER; @@ -871,86 +1380,244 @@ again: rv = http_parse_header(conn, buf); free(buf); + if (rv == -1) - return -1; - if (rv == -2) /* redirect */ - return 0; + return http_failed(conn); if (rv == 0) done = 1; } /* Check status header and decide what to do next */ - if (conn->status == 200) { + if (http_isok(conn) || http_isredirect(conn)) { + if (http_isredirect(conn)) + http_redirect(conn); + + conn->totalsz = 0; if (conn->chunked) - conn->state = STATE_RESPONSE_CHUNKED; + conn->state = STATE_RESPONSE_CHUNKED_HEADER; else conn->state = STATE_RESPONSE_DATA; goto again; + } else if (conn->status == 100 || conn->status == 103) { + conn->state = STATE_RESPONSE_STATUS; } else if (conn->status == 304) { - http_done(conn->id, HTTP_NOT_MOD, conn->last_modified); - } else { - http_done(conn->id, HTTP_FAILED, conn->last_modified); + return http_done(conn, HTTP_NOT_MOD); } - - conn->state = STATE_DONE; - return 0; + + return http_failed(conn); case STATE_RESPONSE_DATA: - if (conn->bufpos == conn->bufsz || - conn->iosz <= (off_t)conn->bufpos) - return 0; - goto read_more; - case STATE_RESPONSE_CHUNKED: - while (conn->iosz == 0) { - buf = http_get_line(conn); - if (buf == NULL) - goto read_more; - switch (http_parse_chunked(conn, buf)) { - case -1: - free(buf); - return -1; - case 0: - free(buf); - return 0; + if (conn->bufpos != conn->bufsz && + conn->iosz > (off_t)conn->bufpos) + goto read_more; + + /* got a full buffer full of data */ + if (conn->req == NULL) { + /* + * After redirects all data needs to be discarded. + */ + if (conn->iosz < (off_t)conn->bufpos) { + conn->bufpos -= conn->iosz; + conn->iosz = 0; + } else { + conn->iosz -= conn->bufpos; + conn->bufpos = 0; } + if (conn->chunked) + conn->state = STATE_RESPONSE_CHUNKED_TRAILER; + else + conn->state = STATE_RESPONSE_DATA; + goto read_more; + } + + conn->state = STATE_WRITE_DATA; + return WANT_POLLOUT; + case STATE_RESPONSE_CHUNKED_HEADER: + assert(conn->iosz == 0); + + buf = http_get_line(conn); + if (buf == NULL) + goto read_more; + if (http_parse_chunked(conn, buf) != 0) { + warnx("%s: bad chunk encoding", http_info(conn->host)); + free(buf); + return http_failed(conn); + } + free(buf); + + /* + * check if transfer is done, in which case the last trailer + * still needs to be processed. + */ + if (conn->iosz == 0) { + conn->chunked = 0; + conn->state = STATE_RESPONSE_CHUNKED_TRAILER; + goto again; + } + + conn->state = STATE_RESPONSE_DATA; + goto again; + case STATE_RESPONSE_CHUNKED_TRAILER: + buf = http_get_line(conn); + if (buf == NULL) + goto read_more; + if (http_parse_chunked(conn, buf) != 1) { + warnx("%s: bad chunk encoding", http_info(conn->host)); free(buf); + return http_failed(conn); } + free(buf); - if (conn->bufpos == conn->bufsz || - conn->iosz <= (off_t)conn->bufpos) - return 0; - goto read_more; + /* if chunked got cleared then the transfer is over */ + if (conn->chunked == 0) + return http_done(conn, HTTP_OK); + + conn->state = STATE_RESPONSE_CHUNKED_HEADER; + goto again; default: errx(1, "unexpected http state"); } } -static int +/* + * Send out the HTTP request. When done, replace buffer with the read buffer. + */ +static enum res http_write(struct http_connection *conn) { ssize_t s; - s = tls_write(conn->tls, conn->buf + conn->bufpos, + assert(conn->state == STATE_REQUEST); + + while (conn->bufpos < conn->bufsz) { + s = tls_write(conn->tls, conn->buf + conn->bufpos, + conn->bufsz - conn->bufpos); + if (s == -1) { + warnx("%s: TLS write: %s", http_info(conn->host), + tls_error(conn->tls)); + return http_failed(conn); + } else if (s == TLS_WANT_POLLIN) { + return WANT_POLLIN; + } else if (s == TLS_WANT_POLLOUT) { + return WANT_POLLOUT; + } + + conn->bufpos += s; + } + + /* done writing, first thing we need the status */ + conn->state = STATE_RESPONSE_STATUS; + + /* free write buffer and allocate the read buffer */ + free(conn->buf); + conn->bufpos = 0; + conn->bufsz = HTTP_BUF_SIZE; + if ((conn->buf = malloc(conn->bufsz)) == NULL) + err(1, NULL); + + return http_read(conn); +} + +static enum res +proxy_read(struct http_connection *conn) +{ + ssize_t s; + char *buf; + int done; + + s = read(conn->fd, conn->buf + conn->bufpos, conn->bufsz - conn->bufpos); if (s == -1) { - warnx("%s: TLS write: %s", http_info(conn->url), - tls_error(conn->tls)); - return -1; - } else if (s == TLS_WANT_POLLIN) { - return WANT_POLLIN; - } else if (s == TLS_WANT_POLLOUT) { - return WANT_POLLOUT; + warn("%s: read", http_info(conn->host)); + return http_failed(conn); + } + + if (s == 0) { + if (conn->req) + warnx("%s: short read, connection closed", + http_info(conn->host)); + return http_failed(conn); } conn->bufpos += s; - if (conn->bufpos == conn->bufsz) - return 0; - return WANT_POLLOUT; +again: + switch (conn->state) { + case STATE_PROXY_STATUS: + buf = http_get_line(conn); + if (buf == NULL) + return WANT_POLLIN; + if (http_parse_status(conn, buf) == -1) { + free(buf); + return http_failed(conn); + } + free(buf); + conn->state = STATE_PROXY_RESPONSE; + goto again; + case STATE_PROXY_RESPONSE: + done = 0; + while (!done) { + buf = http_get_line(conn); + if (buf == NULL) + return WANT_POLLIN; + /* empty line, end of header */ + if (*buf == '\0') + done = 1; + free(buf); + } + /* proxy is ready, connect to remote */ + if (conn->status == 200) { + conn->state = STATE_CONNECT; + return http_tls_connect(conn); + } + return http_failed(conn); + default: + errx(1, "unexpected http state"); + } } -static int +/* + * Send out the proxy request. When done, replace buffer with the read buffer. + */ +static enum res +proxy_write(struct http_connection *conn) +{ + ssize_t s; + + assert(conn->state == STATE_PROXY_REQUEST); + + s = write(conn->fd, conn->buf + conn->bufpos, + conn->bufsz - conn->bufpos); + if (s == -1) { + warn("%s: write", http_info(conn->host)); + return http_failed(conn); + } + conn->bufpos += s; + if (conn->bufpos < conn->bufsz) + return WANT_POLLOUT; + + /* done writing, first thing we need the status */ + conn->state = STATE_PROXY_STATUS; + + /* free write buffer and allocate the read buffer */ + free(conn->buf); + conn->bufpos = 0; + conn->bufsz = HTTP_BUF_SIZE; + if ((conn->buf = malloc(conn->bufsz)) == NULL) + err(1, NULL); + + return WANT_POLLIN; +} + +/* + * Properly shutdown the TLS session else move connection into free state. + */ +static enum res http_close(struct http_connection *conn) { + assert(conn->state == STATE_IDLE || conn->state == STATE_CLOSE); + + conn->state = STATE_CLOSE; + if (conn->tls != NULL) { switch (tls_close(conn->tls)) { case TLS_WANT_POLLIN: @@ -963,22 +1630,35 @@ http_close(struct http_connection *conn) } } - return -1; + conn->state = STATE_FREE; + return DONE; } -static int +/* + * Write data into provided file descriptor. If all data got written + * the connection may change into idle state. + */ +static enum res data_write(struct http_connection *conn) { ssize_t s; size_t bsz = conn->bufpos; + assert(conn->state == STATE_WRITE_DATA); + if (conn->iosz < (off_t)bsz) bsz = conn->iosz; - s = write(conn->outfd, conn->buf, bsz); + s = write(conn->req->outfd, conn->buf, bsz); if (s == -1) { - warn("%s: data write", http_info(conn->url)); - return -1; + warn("%s: data write", http_info(conn->req->uri)); + return http_failed(conn); + } + + conn->totalsz += s; + if (conn->totalsz > MAX_CONTENTLEN) { + warn("%s: too much data offered", http_info(conn->req->uri)); + return http_failed(conn); } conn->bufpos -= s; @@ -986,16 +1666,13 @@ data_write(struct http_connection *conn) memmove(conn->buf, conn->buf + s, conn->bufpos); /* check if regular file transfer is finished */ - if (!conn->chunked && conn->iosz == 0) { - http_done(conn->id, HTTP_OK, conn->last_modified); - conn->state = STATE_DONE; - return 0; - } + if (!conn->chunked && conn->iosz == 0) + return http_done(conn, HTTP_OK); /* all data written, switch back to read */ if (conn->bufpos == 0 || conn->iosz == 0) { - if (conn->chunked) - conn->state = STATE_RESPONSE_CHUNKED; + if (conn->chunked && conn->iosz == 0) + conn->state = STATE_RESPONSE_CHUNKED_TRAILER; else conn->state = STATE_RESPONSE_DATA; return http_read(conn); @@ -1011,30 +1688,38 @@ data_write(struct http_connection *conn) * If 0 is returned this stage is finished and the protocol should move * to the next stage by calling http_nextstep(). On error return -1. */ -static int -http_handle(struct http_connection *conn, int events) +static enum res +http_handle(struct http_connection *conn) { + assert (conn->pfd != NULL && conn->pfd->revents != 0); + + conn->io_time = 0; + switch (conn->state) { - case STATE_INIT: - return http_connect(conn); case STATE_CONNECT: - if (http_finish_connect(conn) == -1) - /* something went wrong, try other host */ - return http_connect(conn); - return 0; + return http_finish_connect(conn); case STATE_TLSCONNECT: return http_tls_handshake(conn); case STATE_REQUEST: return http_write(conn); + case STATE_PROXY_REQUEST: + return proxy_write(conn); + case STATE_PROXY_STATUS: + case STATE_PROXY_RESPONSE: + return proxy_read(conn); case STATE_RESPONSE_STATUS: case STATE_RESPONSE_HEADER: case STATE_RESPONSE_DATA: - case STATE_RESPONSE_CHUNKED: + case STATE_RESPONSE_CHUNKED_HEADER: + case STATE_RESPONSE_CHUNKED_TRAILER: return http_read(conn); case STATE_WRITE_DATA: return data_write(conn); - case STATE_DONE: + case STATE_CLOSE: return http_close(conn); + case STATE_IDLE: + conn->state = STATE_RESPONSE_HEADER; + return http_read(conn); case STATE_FREE: errx(1, "bad http state"); } @@ -1042,94 +1727,17 @@ http_handle(struct http_connection *conn } /* - * Move the state machine forward until IO needs to happen. - * Returns either WANT_POLLIN or WANT_POLLOUT or -1 on error. + * Initialisation done before pledge() call to load certificates. */ -static int -http_nextstep(struct http_connection *conn) -{ - int r; - - switch (conn->state) { - case STATE_INIT: - return http_connect(conn); - case STATE_CONNECT: - conn->state = STATE_TLSCONNECT; - r = http_tls_connect(conn); - if (r != 0) - return r; - /* FALLTHROUGH */ - case STATE_TLSCONNECT: - conn->state = STATE_REQUEST; - return http_request(conn); - case STATE_REQUEST: - conn->state = STATE_RESPONSE_STATUS; - free(conn->buf); - /* allocate the read buffer */ - if ((conn->buf = malloc(HTTP_BUF_SIZE)) == NULL) - err(1, NULL); - conn->bufpos = 0; - conn->bufsz = HTTP_BUF_SIZE; - return http_read(conn); - case STATE_RESPONSE_DATA: - case STATE_RESPONSE_CHUNKED: - conn->state = STATE_WRITE_DATA; - return WANT_POLLOUT; - case STATE_DONE: - return http_close(conn); - case STATE_RESPONSE_STATUS: - case STATE_RESPONSE_HEADER: - case STATE_WRITE_DATA: - case STATE_FREE: - errx(1, "bad http state"); - } - errx(1, "unknown http state"); -} - -static int -http_do(struct http_connection *conn, int events) -{ - switch (http_handle(conn, events)) { - case -1: - /* connection failure */ - if (conn->state != STATE_DONE) - http_fail(conn->id); - http_free(conn); - return -1; - case 0: - switch (http_nextstep(conn)) { - case WANT_POLLIN: - conn->events = POLLIN; - break; - case WANT_POLLOUT: - conn->events = POLLOUT; - break; - case -1: - if (conn->state != STATE_DONE) - http_fail(conn->id); - http_free(conn); - return -1; - case 0: - errx(1, "%s: http_nextstep returned 0, state %d", - http_info(conn->url), conn->state); - } - break; - case WANT_POLLIN: - conn->events = POLLIN; - break; - case WANT_POLLOUT: - conn->events = POLLOUT; - break; - } - return 0; -} - static void http_setup(void) { + char *httpproxy; + tls_config = tls_config_new(); if (tls_config == NULL) errx(1, "tls config failed"); + #if 0 /* TODO Should we allow extra protos and ciphers? */ if (tls_config_set_protocols(tls_config, TLS_PROTOCOLS_ALL) == -1) @@ -1147,17 +1755,19 @@ http_setup(void) err(1, "tls_load_file: %s", tls_default_ca_cert_file()); tls_config_set_ca_mem(tls_config, tls_ca_mem, tls_ca_size); - /* TODO initalize proxy settings */ + if ((httpproxy = getenv("http_proxy")) != NULL && *httpproxy == '\0') + httpproxy = NULL; + proxy_parse_uri(httpproxy); } void proc_http(char *bind_addr, int fd) { - struct http_connection *http_conns[MAX_CONNECTIONS]; - struct pollfd pfds[MAX_CONNECTIONS + 1]; - size_t i; - int active_connections; + struct pollfd pfds[NPFDS]; + struct http_connection *conn, *nc; + struct http_request *req, *nr; + struct ibuf *b, *inbuf = NULL; if (bind_addr != NULL) { struct addrinfo hints, *res; @@ -1176,42 +1786,70 @@ proc_http(char *bind_addr, int fd) if (pledge("stdio inet dns recvfd", NULL) == -1) err(1, "pledge"); - memset(&http_conns, 0, sizeof(http_conns)); memset(&pfds, 0, sizeof(pfds)); - pfds[MAX_CONNECTIONS].fd = fd; msgbuf_init(&msgq); msgq.fd = fd; for (;;) { - active_connections = 0; - for (i = 0; i < MAX_CONNECTIONS; i++) { - struct http_connection *conn = http_conns[i]; + time_t now; + int timeout; + size_t i; - if (conn == NULL) { - pfds[i].fd = -1; - continue; + pfds[0].fd = fd; + pfds[0].events = POLLIN; + if (msgq.queued) + pfds[0].events |= POLLOUT; + + i = 1; + timeout = INFTIM; + now = getmonotime(); + LIST_FOREACH(conn, &active, entry) { + if (conn->io_time == 0) + conn->io_time = now + HTTP_IO_TIMEOUT; + + if (conn->io_time <= now) + timeout = 0; + else { + int diff = conn->io_time - now; + diff *= 1000; + if (timeout == INFTIM || diff < timeout) + timeout = diff; } if (conn->state == STATE_WRITE_DATA) - pfds[i].fd = conn->outfd; + pfds[i].fd = conn->req->outfd; else pfds[i].fd = conn->fd; pfds[i].events = conn->events; - active_connections++; + conn->pfd = &pfds[i]; + i++; + if (i > NPFDS) + errx(1, "too many connections"); + } + LIST_FOREACH(conn, &idle, entry) { + if (conn->idle_time <= now) + timeout = 0; + else { + int diff = conn->idle_time - now; + diff *= 1000; + if (timeout == INFTIM || diff < timeout) + timeout = diff; + } + pfds[i].fd = conn->fd; + pfds[i].events = POLLIN; + conn->pfd = &pfds[i]; + i++; + if (i > NPFDS) + errx(1, "too many connections"); } - pfds[MAX_CONNECTIONS].events = 0; - if (active_connections < MAX_CONNECTIONS) - pfds[MAX_CONNECTIONS].events |= POLLIN; - if (msgq.queued) - pfds[MAX_CONNECTIONS].events |= POLLOUT; - if (poll(pfds, sizeof(pfds) / sizeof(pfds[0]), INFTIM) == -1) + if (poll(pfds, i, timeout) == -1) err(1, "poll"); - if (pfds[MAX_CONNECTIONS].revents & POLLHUP) + if (pfds[0].revents & POLLHUP) break; - if (pfds[MAX_CONNECTIONS].revents & POLLOUT) { + if (pfds[0].revents & POLLOUT) { switch (msgbuf_write(&msgq)) { case 0: errx(1, "write: connection closed"); @@ -1219,45 +1857,54 @@ proc_http(char *bind_addr, int fd) err(1, "write"); } } + if (pfds[0].revents & POLLIN) { + b = io_buf_recvfd(fd, &inbuf); + if (b != NULL) { + size_t id; + char *uri; + char *mod; + + io_read_buf(b, &id, sizeof(id)); + io_read_str(b, &uri); + io_read_str(b, &mod); + + /* queue up new requests */ + http_req_new(id, uri, mod, 0, b->fd); + ibuf_free(b); + } + } - /* process active http requests */ - for (i = 0; i < MAX_CONNECTIONS; i++) { - struct http_connection *conn = http_conns[i]; - - if (conn == NULL) - continue; - /* event not ready */ - if (pfds[i].revents == 0) - continue; + now = getmonotime(); + /* process idle connections */ + LIST_FOREACH_SAFE(conn, &idle, entry, nc) { + if (conn->pfd != NULL && conn->pfd->revents != 0) + http_do(conn, http_handle); + else if (conn->idle_time <= now) + http_do(conn, http_close); - if (http_do(conn, pfds[i].revents) == -1) - http_conns[i] = NULL; + if (conn->state == STATE_FREE) + http_free(conn); } - /* process new requests last */ - if (pfds[MAX_CONNECTIONS].revents & POLLIN) { - struct http_connection *h; - size_t id; - int outfd; - char *uri; - char *mod; - - outfd = io_recvfd(fd, &id, sizeof(id)); - io_str_read(fd, &uri); - io_str_read(fd, &mod); - - h = http_new(id, uri, mod, outfd); - if (h != NULL) { - for (i = 0; i < MAX_CONNECTIONS; i++) { - if (http_conns[i] != NULL) - continue; - http_conns[i] = h; - if (http_do(h, 0) == -1) - http_conns[i] = NULL; - break; - } + /* then active http requests */ + LIST_FOREACH_SAFE(conn, &active, entry, nc) { + /* check if event is ready */ + if (conn->pfd != NULL && conn->pfd->revents != 0) + http_do(conn, http_handle); + else if (conn->io_time <= now) { + warnx("%s: timeout, connection closed", + http_info(conn->host)); + http_do(conn, http_failed); } + + if (conn->state == STATE_FREE) + http_free(conn); } + + + TAILQ_FOREACH_SAFE(req, &queue, entry, nr) + if (!http_req_schedule(req)) + break; } exit(0); Index: usr.sbin/rpki-client/io.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/io.c,v retrieving revision 1.13 diff -u -p -u -r1.13 io.c --- usr.sbin/rpki-client/io.c 4 Mar 2021 13:01:41 -0000 1.13 +++ usr.sbin/rpki-client/io.c 6 Nov 2021 18:21:54 -0000 @@ -1,5 +1,7 @@ /* $OpenBSD: io.c,v 1.13 2021/03/04 13:01:41 claudio Exp $ */ + /* + * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons * * Permission to use, copy, modify, and distribute this software for any @@ -30,30 +32,23 @@ #include "extern.h" -void -io_socket_blocking(int fd) -{ - int fl; - - if ((fl = fcntl(fd, F_GETFL, 0)) == -1) - err(1, "fcntl"); - if (fcntl(fd, F_SETFL, fl & ~O_NONBLOCK) == -1) - err(1, "fcntl"); -} - -void -io_socket_nonblocking(int fd) +/* + * Create new io buffer, call io_close() when done with it. + * Function always returns a new buffer. + */ +struct ibuf * +io_new_buffer(void) { - int fl; + struct ibuf *b; - if ((fl = fcntl(fd, F_GETFL, 0)) == -1) - err(1, "fcntl"); - if (fcntl(fd, F_SETFL, fl | O_NONBLOCK) == -1) - err(1, "fcntl"); + if ((b = ibuf_dynamic(64, INT32_MAX)) == NULL) + err(1, NULL); + ibuf_reserve(b, sizeof(size_t)); /* can not fail */ + return b; } /* - * Like io_simple_write() but into a buffer. + * Add a simple object of static size to the io buffer. */ void io_simple_buffer(struct ibuf *b, const void *res, size_t sz) @@ -87,74 +82,155 @@ io_str_buffer(struct ibuf *b, const char } /* - * Read of a binary buffer that must be on a blocking descriptor. + * Finish and enqueue a io buffer. + */ +void +io_close_buffer(struct msgbuf *msgbuf, struct ibuf *b) +{ + size_t len; + + len = ibuf_size(b) - sizeof(len); + memcpy(ibuf_seek(b, 0, sizeof(len)), &len, sizeof(len)); + ibuf_close(msgbuf, b); +} + +/* + * Read of an ibuf and extract sz byte from there. * Does nothing if "sz" is zero. - * This will fail and exit on EOF. + * Return 1 on success or 0 if there was not enough data. */ void -io_simple_read(int fd, void *res, size_t sz) +io_read_buf(struct ibuf *b, void *res, size_t sz) { - ssize_t ssz; char *tmp; - tmp = res; /* arithmetic on a pointer to void is a GNU extension */ -again: if (sz == 0) return; - if ((ssz = read(fd, tmp, sz)) == -1) - err(1, "read"); - else if (ssz == 0) - errx(1, "read: unexpected end of file"); - else if ((size_t)ssz == sz) + tmp = ibuf_seek(b, b->rpos, sz); + if (tmp == NULL) + errx(1, "bad internal framing, buffer too short"); + b->rpos += sz; + memcpy(res, tmp, sz); +} + +/* + * Read a string (returns NULL for zero-length strings), allocating + * space for it. + * Return 1 on success or 0 if there was not enough data. + */ +void +io_read_str(struct ibuf *b, char **res) +{ + size_t sz; + + io_read_buf(b, &sz, sizeof(sz)); + if (sz == 0) { + *res = NULL; return; - sz -= ssz; - tmp += ssz; - goto again; + } + if ((*res = calloc(sz + 1, 1)) == NULL) + err(1, NULL); + io_read_buf(b, *res, sz); } /* * Read a binary buffer, allocating space for it. * If the buffer is zero-sized, this won't allocate "res", but * will still initialise it to NULL. + * Return 1 on success or 0 if there was not enough data. */ void -io_buf_read_alloc(int fd, void **res, size_t *sz) +io_read_buf_alloc(struct ibuf *b, void **res, size_t *sz) { - *res = NULL; - io_simple_read(fd, sz, sizeof(size_t)); + io_read_buf(b, sz, sizeof(sz)); if (*sz == 0) return; if ((*res = malloc(*sz)) == NULL) err(1, NULL); - io_simple_read(fd, *res, *sz); + io_read_buf(b, *res, *sz); +} + +/* XXX copy from imsg-buffer.c */ +static int +ibuf_realloc(struct ibuf *buf, size_t len) +{ + unsigned char *b; + + /* on static buffers max is eq size and so the following fails */ + if (buf->wpos + len > buf->max) { + errno = ERANGE; + return (-1); + } + + b = recallocarray(buf->buf, buf->size, buf->wpos + len, 1); + if (b == NULL) + return (-1); + buf->buf = b; + buf->size = buf->wpos + len; + + return (0); } /* - * Read a string (returns NULL for zero-length strings), allocating - * space for it. + * Read once and fill a ibuf until it is finished. + * Returns NULL if more data is needed, returns a full ibuf once + * all data is received. */ -void -io_str_read(int fd, char **res) +struct ibuf * +io_buf_read(int fd, struct ibuf **ib) { - size_t sz; + struct ibuf *b = *ib; + ssize_t n; + size_t sz; - io_simple_read(fd, &sz, sizeof(size_t)); - if (sz == 0) { - *res = NULL; - return; + /* if ibuf == NULL allocate a new buffer */ + if (b == NULL) { + if ((b = ibuf_dynamic(sizeof(sz), INT32_MAX)) == NULL) + err(1, NULL); + *ib = b; } - if ((*res = calloc(sz + 1, 1)) == NULL) - err(1, NULL); - io_simple_read(fd, *res, sz); + + /* read some data */ + while ((n = read(fd, b->buf + b->wpos, b->size - b->wpos)) == -1) { + if (errno == EINTR) + continue; + err(1, "read"); + } + + if (n == 0) + errx(1, "read: unexpected end of file"); + b->wpos += n; + + /* got full message */ + if (b->wpos == b->size) { + /* only header received */ + if (b->wpos == sizeof(sz)) { + memcpy(&sz, b->buf, sizeof(sz)); + if (sz == 0 || sz > INT32_MAX) + errx(1, "bad internal framing, bad size"); + if (ibuf_realloc(b, sz) == -1) + err(1, "ibuf_realloc"); + return NULL; + } + + /* skip over initial size header */ + b->rpos += sizeof(sz); + *ib = NULL; + return b; + } + + return NULL; } + /* * Read data from socket but receive a file descriptor at the same time. */ -int -io_recvfd(int fd, void *res, size_t sz) +struct ibuf * +io_buf_recvfd(int fd, struct ibuf **ib) { + struct ibuf *b = *ib; struct iovec iov; struct msghdr msg; struct cmsghdr *cmsg; @@ -162,15 +238,22 @@ io_recvfd(int fd, void *res, size_t sz) struct cmsghdr hdr; char buf[CMSG_SPACE(sizeof(int))]; } cmsgbuf; - int outfd = -1; - char *b = res; ssize_t n; + size_t sz; + + /* fd are only passed on the head, just use regular read afterwards */ + if (b != NULL) + return io_buf_read(fd, ib); + if ((b = ibuf_dynamic(sizeof(sz), INT32_MAX)) == NULL) + err(1, NULL); + *ib = b; + memset(&msg, 0, sizeof(msg)); memset(&cmsgbuf, 0, sizeof(cmsgbuf)); - iov.iov_base = res; - iov.iov_len = sz; + iov.iov_base = b->buf; + iov.iov_len = b->size; msg.msg_iov = &iov; msg.msg_iovlen = 1; @@ -197,29 +280,32 @@ io_recvfd(int fd, void *res, size_t sz) for (i = 0; i < j; i++) { f = ((int *)CMSG_DATA(cmsg))[i]; if (i == 0) - outfd = f; + b->fd = f; else close(f); } } } - b += n; - sz -= n; - while (sz > 0) { - /* short receive */ - n = recv(fd, b, sz, 0); - if (n == -1) { - if (errno == EINTR) - continue; - err(1, "recv"); + b->wpos += n; + + /* got full message */ + if (b->wpos == b->size) { + /* only header received */ + if (b->wpos == sizeof(sz)) { + memcpy(&sz, b->buf, sizeof(sz)); + if (sz == 0 || sz > INT32_MAX) + errx(1, "read: bad internal framing, %zu", sz); + if (ibuf_realloc(b, sz) == -1) + err(1, "ibuf_realloc"); + return NULL; } - if (n == 0) - errx(1, "recv: unexpected end of file"); - b += n; - sz -= n; + /* skip over initial size header */ + b->rpos += sizeof(sz); + *ib = NULL; + return b; } - return outfd; + return NULL; } Index: usr.sbin/rpki-client/ip.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/ip.c,v retrieving revision 1.16 diff -u -p -u -r1.16 ip.c --- usr.sbin/rpki-client/ip.c 29 Mar 2021 06:15:29 -0000 1.16 +++ usr.sbin/rpki-client/ip.c 6 Nov 2021 18:11:59 -0000 @@ -110,7 +110,6 @@ ip_addr_check_overlap(const struct cert_ size_t i, sz = ip->afi == AFI_IPV4 ? 4 : 16; int inherit_v4 = 0, inherit_v6 = 0; int has_v4 = 0, has_v6 = 0, socktype; - char buf[64]; /* * FIXME: cache this by having a flag on the cert_ip, else we're @@ -147,6 +146,8 @@ ip_addr_check_overlap(const struct cert_ /* Check our ranges. */ for (i = 0; i < ipsz; i++) { + char buf[64]; + if (ips[i].afi != ip->afi) continue; if (memcmp(ips[i].max, ip->min, sz) <= 0 || @@ -281,58 +282,6 @@ ip_addr_print(const struct ip_addr *addr ip4_addr2str(addr, buf, bufsz); else ip6_addr2str(addr, buf, bufsz); -} - -/* - * Serialise an ip_addr for sending over the wire. - * Matched with ip_addr_read(). - */ -void -ip_addr_buffer(struct ibuf *b, const struct ip_addr *p) -{ - size_t sz = PREFIX_SIZE(p->prefixlen); - - assert(sz <= 16); - io_simple_buffer(b, &p->prefixlen, sizeof(unsigned char)); - io_simple_buffer(b, p->addr, sz); -} - -/* - * Serialise an ip_addr_range for sending over the wire. - * Matched with ip_addr_range_read(). - */ -void -ip_addr_range_buffer(struct ibuf *b, const struct ip_addr_range *p) -{ - ip_addr_buffer(b, &p->min); - ip_addr_buffer(b, &p->max); -} - -/* - * Read an ip_addr from the wire. - * Matched with ip_addr_buffer(). - */ -void -ip_addr_read(int fd, struct ip_addr *p) -{ - size_t sz; - - io_simple_read(fd, &p->prefixlen, sizeof(unsigned char)); - sz = PREFIX_SIZE(p->prefixlen); - assert(sz <= 16); - io_simple_read(fd, p->addr, sz); -} - -/* - * Read an ip_addr_range from the wire. - * Matched with ip_addr_range_buffer(). - */ -void -ip_addr_range_read(int fd, struct ip_addr_range *p) -{ - - ip_addr_read(fd, &p->min); - ip_addr_read(fd, &p->max); } /* Index: usr.sbin/rpki-client/main.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/main.c,v retrieving revision 1.138 diff -u -p -u -r1.138 main.c --- usr.sbin/rpki-client/main.c 15 Apr 2021 14:22:05 -0000 1.138 +++ usr.sbin/rpki-client/main.c 6 Nov 2021 18:12:15 -0000 @@ -1,5 +1,6 @@ /* $OpenBSD: main.c,v 1.138 2021/04/15 14:22:05 claudio Exp $ */ /* + * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons * * Permission to use, copy, modify, and distribute this software for any @@ -15,11 +16,12 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +#include #include #include #include +#include #include -#include #include #include @@ -47,6 +49,11 @@ */ #define TALSZ_MAX 8 +const char *tals[TALSZ_MAX]; +const char *taldescs[TALSZ_MAX]; +unsigned int talrepocnt[TALSZ_MAX]; +size_t talsz; + size_t entity_queue; int timeout = 60*60; volatile sig_atomic_t killme; @@ -60,7 +67,7 @@ const char *bird_tablename = "ROAS"; int verbose; int noop; -int rrdpon; +int rrdpon = 1; struct stats stats; @@ -80,16 +87,24 @@ logx(const char *fmt, ...) } } +time_t +getmonotime(void) +{ + struct timespec ts; + + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) + err(1, "clock_gettime"); + return (ts.tv_sec); +} + void entity_free(struct entity *ent) { - if (ent == NULL) return; - free(ent->pkey); + free(ent->data); free(ent->file); - free(ent->descr); free(ent); } @@ -99,15 +114,14 @@ entity_free(struct entity *ent) * The pointer must be passed entity_free(). */ void -entity_read_req(int fd, struct entity *ent) +entity_read_req(struct ibuf *b, struct entity *ent) { - - io_simple_read(fd, &ent->type, sizeof(enum rtype)); - io_str_read(fd, &ent->file); - io_simple_read(fd, &ent->has_pkey, sizeof(int)); - if (ent->has_pkey) - io_buf_read_alloc(fd, (void **)&ent->pkey, &ent->pkeysz); - io_str_read(fd, &ent->descr); + io_read_buf(b, &ent->type, sizeof(ent->type)); + io_read_buf(b, &ent->talid, sizeof(ent->talid)); + io_read_str(b, &ent->file); + io_read_buf(b, &ent->has_data, sizeof(ent->has_data)); + if (ent->has_data) + io_read_buf_alloc(b, (void **)&ent->data, &ent->datasz); } /* @@ -121,18 +135,18 @@ entity_write_req(const struct entity *en if (filepath_add(&fpt, ent->file) == 0) { warnx("%s: File already visited", ent->file); + entity_queue--; return; } - if ((b = ibuf_dynamic(sizeof(*ent), UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &ent->type, sizeof(ent->type)); + io_simple_buffer(b, &ent->talid, sizeof(ent->talid)); io_str_buffer(b, ent->file); - io_simple_buffer(b, &ent->has_pkey, sizeof(int)); - if (ent->has_pkey) - io_buf_buffer(b, ent->pkey, ent->pkeysz); - io_str_buffer(b, ent->descr); - ibuf_close(&procq, b); + io_simple_buffer(b, &ent->has_data, sizeof(int)); + if (ent->has_data) + io_buf_buffer(b, ent->data, ent->datasz); + io_close_buffer(&procq, b); } /* @@ -145,13 +159,14 @@ entityq_flush(struct entityq *q, struct struct entity *p, *np; TAILQ_FOREACH_SAFE(p, q, entries, np) { + char *file = p->file; + /* * XXX fixup path here since the repo may change * during load because of fallback. In that case * the file path changes as well since RRDP and RSYNC * can not share a common repo. */ - char *file = p->file; p->file = repo_filename(rp, file); if (p->file == NULL) err(1, "can't construct repo filename"); @@ -168,7 +183,7 @@ entityq_flush(struct entityq *q, struct */ static void entityq_add(char *file, enum rtype type, struct repo *rp, - const unsigned char *pkey, size_t pkeysz, char *descr) + unsigned char *data, size_t datasz, int talid) { struct entity *p; @@ -176,17 +191,13 @@ entityq_add(char *file, enum rtype type, err(1, NULL); p->type = type; + p->talid = talid; p->file = file; - p->has_pkey = pkey != NULL; - if (p->has_pkey) { - p->pkeysz = pkeysz; - if ((p->pkey = malloc(pkeysz)) == NULL) - err(1, NULL); - memcpy(p->pkey, pkey, pkeysz); + p->has_data = data != NULL; + if (p->has_data) { + p->data = data; + p->datasz = datasz; } - if (descr != NULL) - if ((p->descr = strdup(descr)) == NULL) - err(1, NULL); entity_queue++; @@ -221,12 +232,11 @@ rrdp_file_resp(size_t id, int ok) enum rrdp_msg type = RRDP_FILE; struct ibuf *b; - if ((b = ibuf_open(sizeof(type) + sizeof(id) + sizeof(ok))) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &id, sizeof(id)); io_simple_buffer(b, &ok, sizeof(ok)); - ibuf_close(&rrdpq, b); + io_close_buffer(&rrdpq, b); } void @@ -236,8 +246,7 @@ rrdp_fetch(size_t id, const char *uri, c enum rrdp_msg type = RRDP_START; struct ibuf *b; - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &id, sizeof(id)); io_str_buffer(b, local); @@ -245,7 +254,7 @@ rrdp_fetch(size_t id, const char *uri, c io_str_buffer(b, s->session_id); io_simple_buffer(b, &s->serial, sizeof(s->serial)); io_str_buffer(b, s->last_mod); - ibuf_close(&rrdpq, b); + io_close_buffer(&rrdpq, b); } /* @@ -256,12 +265,11 @@ rsync_fetch(size_t id, const char *uri, { struct ibuf *b; - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &id, sizeof(id)); io_str_buffer(b, local); io_str_buffer(b, uri); - ibuf_close(&rsyncq, b); + io_close_buffer(&rsyncq, b); } /* @@ -272,14 +280,13 @@ http_fetch(size_t id, const char *uri, c { struct ibuf *b; - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &id, sizeof(id)); io_str_buffer(b, uri); io_str_buffer(b, last_mod); /* pass file as fd */ b->fd = fd; - ibuf_close(&httpq, b); + io_close_buffer(&httpq, b); } /* @@ -296,12 +303,11 @@ rrdp_http_fetch(size_t id, const char *u if (pipe2(pi, O_CLOEXEC | O_NONBLOCK) == -1) err(1, "pipe"); - if ((b = ibuf_open(sizeof(type) + sizeof(id))) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &id, sizeof(id)); b->fd = pi[0]; - ibuf_close(&rrdpq, b); + io_close_buffer(&rrdpq, b); http_fetch(id, uri, last_mod, pi[1]); } @@ -313,13 +319,12 @@ rrdp_http_done(size_t id, enum http_resu struct ibuf *b; /* RRDP request, relay response over to the rrdp process */ - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &id, sizeof(id)); io_simple_buffer(b, &res, sizeof(res)); io_str_buffer(b, last_mod); - ibuf_close(&rrdpq, b); + io_close_buffer(&rrdpq, b); } /* @@ -343,7 +348,7 @@ queue_add_from_mft(const char *mft, cons * that the repository has already been loaded. */ - entityq_add(nfile, type, NULL, NULL, 0, NULL); + entityq_add(nfile, type, NULL, NULL, 0, -1); } /* @@ -391,30 +396,22 @@ queue_add_from_mft_set(const struct mft * Add a local TAL file (RFC 7730) to the queue of files to fetch. */ static void -queue_add_tal(const char *file) +queue_add_tal(const char *file, int id) { - char *nfile, *buf; + unsigned char *buf; + char *nfile; + size_t len; if ((nfile = strdup(file)) == NULL) err(1, NULL); - buf = tal_read_file(file); - - /* Record tal for later reporting */ - if (stats.talnames == NULL) { - if ((stats.talnames = strdup(file)) == NULL) - err(1, NULL); - } else { - char *tmp; - if (asprintf(&tmp, "%s %s", stats.talnames, file) == -1) - err(1, NULL); - free(stats.talnames); - stats.talnames = tmp; + buf = load_file(file, &len); + if (buf == NULL) { + warn("%s", file); + return; } /* Not in a repository, so directly add to queue. */ - entityq_add(nfile, RTYPE_TAL, NULL, NULL, 0, buf); - /* entityq_add makes a copy of buf */ - free(buf); + entityq_add(nfile, RTYPE_TAL, NULL, buf, len, id); } /* @@ -424,14 +421,23 @@ static void queue_add_from_tal(struct tal *tal) { struct repo *repo; + unsigned char *data; assert(tal->urisz); + if ((taldescs[tal->id] = strdup(tal->descr)) == NULL) + err(1, NULL); + /* Look up the repository. */ - repo = ta_lookup(tal); + repo = ta_lookup(tal->id, tal); + if (repo == NULL) + return; - entityq_add(NULL, RTYPE_CER, repo, tal->pkey, - tal->pkeysz, tal->descr); + /* steal the pkey from the tal structure */ + data = tal->pkey; + tal->pkey = NULL; + entityq_add(NULL, RTYPE_CER, repo, data, + tal->pkeysz, tal->id); } /* @@ -443,15 +449,14 @@ queue_add_from_cert(const struct cert *c struct repo *repo; char *nfile; - repo = repo_lookup(cert->repo, rrdpon ? cert->notify : NULL); - if (repo == NULL) { - warnx("%s: repository lookup failed", cert->repo); + repo = repo_lookup(cert->talid, cert->repo, + rrdpon ? cert->notify : NULL); + if (repo == NULL) return; - } if ((nfile = strdup(cert->mft)) == NULL) err(1, NULL); - entityq_add(nfile, RTYPE_MFT, repo, NULL, 0, NULL); + entityq_add(nfile, RTYPE_MFT, repo, NULL, 0, -1); } /* @@ -461,9 +466,10 @@ queue_add_from_cert(const struct cert *c * In all cases, we gather statistics. */ static void -entity_process(int proc, struct stats *st, struct vrp_tree *tree) +entity_process(struct ibuf *b, struct stats *st, struct vrp_tree *tree, + struct brk_tree *brktree) { - enum rtype type; + enum rtype type; struct tal *tal; struct cert *cert; struct mft *mft; @@ -476,24 +482,24 @@ entity_process(int proc, struct stats *s * certificate, for example). * We follow that up with whether the resources didn't parse. */ - io_simple_read(proc, &type, sizeof(type)); + io_read_buf(b, &type, sizeof(type)); switch (type) { case RTYPE_TAL: st->tals++; - tal = tal_read(proc); + tal = tal_read(b); queue_add_from_tal(tal); tal_free(tal); break; case RTYPE_CER: st->certs++; - io_simple_read(proc, &c, sizeof(int)); + io_read_buf(b, &c, sizeof(c)); if (c == 0) { st->certs_fail++; break; } - cert = cert_read(proc); - if (cert->valid) { + cert = cert_read(b); + if (cert->purpose == CERT_PURPOSE_CA) { /* * Process the revocation list from the * certificate *first*, since it might mark that @@ -501,18 +507,21 @@ entity_process(int proc, struct stats *s * process the MFT. */ queue_add_from_cert(cert); + } else if (cert->purpose == CERT_PURPOSE_BGPSEC_ROUTER) { + cert_insert_brks(brktree, cert); + st->brks++; } else - st->certs_invalid++; + st->certs_fail++; cert_free(cert); break; case RTYPE_MFT: st->mfts++; - io_simple_read(proc, &c, sizeof(int)); + io_read_buf(b, &c, sizeof(c)); if (c == 0) { st->mfts_fail++; break; } - mft = mft_read(proc); + mft = mft_read(b); if (mft->stale) st->mfts_stale++; queue_add_from_mft_set(mft); @@ -523,12 +532,12 @@ entity_process(int proc, struct stats *s break; case RTYPE_ROA: st->roas++; - io_simple_read(proc, &c, sizeof(int)); + io_read_buf(b, &c, sizeof(c)); if (c == 0) { st->roas_fail++; break; } - roa = roa_read(proc); + roa = roa_read(b); if (roa->valid) roa_insert_vrps(tree, roa, &st->vrps, &st->uniqs); else @@ -539,12 +548,63 @@ entity_process(int proc, struct stats *s st->gbrs++; break; default: - errx(1, "unknown entity type"); + errx(1, "unknown entity type %d", type); } entity_queue--; } +static void +rrdp_process(struct ibuf *b) +{ + enum rrdp_msg type; + enum publish_type pt; + struct rrdp_session s; + char *uri, *last_mod, *data; + char hash[SHA256_DIGEST_LENGTH]; + size_t dsz, id; + int ok; + + io_read_buf(b, &type, sizeof(type)); + io_read_buf(b, &id, sizeof(id)); + + switch (type) { + case RRDP_END: + io_read_buf(b, &ok, sizeof(ok)); + rrdp_finish(id, ok); + break; + case RRDP_HTTP_REQ: + io_read_str(b, &uri); + io_read_str(b, &last_mod); + rrdp_http_fetch(id, uri, last_mod); + break; + case RRDP_SESSION: + io_read_str(b, &s.session_id); + io_read_buf(b, &s.serial, sizeof(s.serial)); + io_read_str(b, &s.last_mod); + rrdp_save_state(id, &s); + free(s.session_id); + free(s.last_mod); + break; + case RRDP_FILE: + io_read_buf(b, &pt, sizeof(pt)); + if (pt != PUB_ADD) + io_read_buf(b, &hash, sizeof(hash)); + io_read_str(b, &uri); + io_read_buf_alloc(b, (void **)&data, &dsz); + + ok = rrdp_handle_file(id, pt, uri, hash, sizeof(hash), + data, dsz); + rrdp_file_resp(id, ok); + + free(uri); + free(data); + break; + default: + errx(1, "unexpected rrdp response"); + } +} + /* * Assign filenames ending in ".tal" in "/etc/rpki" into "tals", * returning the number of files found and filled-in. @@ -552,7 +612,7 @@ entity_process(int proc, struct stats *s * Don't exceded "max" filenames. */ static size_t -tal_load_default(const char *tals[], size_t max) +tal_load_default(void) { static const char *confdir = "/etc/rpki"; size_t s = 0; @@ -566,7 +626,7 @@ tal_load_default(const char *tals[], siz while ((dp = readdir(dirp)) != NULL) { if (fnmatch("*.tal", dp->d_name, FNM_PERIOD) == FNM_NOMATCH) continue; - if (s >= max) + if (s >= TALSZ_MAX) err(1, "too many tal files found in %s", confdir); if (asprintf(&path, "%s/%s", confdir, dp->d_name) == -1) @@ -577,6 +637,31 @@ tal_load_default(const char *tals[], siz return s; } +static void +check_fs_size(int fd, const char *cachedir) +{ + struct statvfs fs; + const long long minsize = 500 * 1024 * 1024; + const long long minnode = 300 * 1000; + + if (fstatvfs(fd, &fs) == -1) + err(1, "statfs %s", cachedir); + + if (fs.f_bavail < minsize / fs.f_frsize || fs.f_favail < minnode) { + fprintf(stderr, "WARNING: rpki-client may need more than " + "the availabe disk space\n" + "on the file-system holding %s.\n", cachedir); + fprintf(stderr, "available space: %lldkB, " + "suggested minimum %lldkB\n", + (long long)fs.f_bavail * fs.f_frsize / 1024, + minsize / 1024); + fprintf(stderr, "available inodes %lld, " + "suggested minimum %lld\n\n", + (long long)fs.f_favail, minnode); + fflush(stderr); + } +} + void suicide(int sig __attribute__((unused))) { @@ -588,19 +673,22 @@ suicide(int sig __attribute__((unused))) int main(int argc, char *argv[]) { - int rc, c, st, proc, rsync, http, rrdp, ok, - hangup = 0, fl = SOCK_STREAM | SOCK_CLOEXEC; - size_t i, id, outsz = 0, talsz = 0; + int rc, c, st, proc, rsync, http, rrdp, ok, hangup = 0; + int fl = SOCK_STREAM | SOCK_CLOEXEC | SOCK_NONBLOCK; + size_t i, id; pid_t pid, procpid, rsyncpid, httppid, rrdppid; int fd[2]; struct pollfd pfd[NPFD]; struct msgbuf *queues[NPFD]; - struct roa **out = NULL; + struct ibuf *b, *httpbuf = NULL, *procbuf = NULL; + struct ibuf *rrdpbuf = NULL, *rsyncbuf = NULL; char *rsync_prog = "openrsync"; char *bind_addr = NULL; const char *cachedir = NULL, *outputdir = NULL; - const char *tals[TALSZ_MAX], *errs, *name; - struct vrp_tree v = RB_INITIALIZER(&v); + const char *errs, *name; + const char *file = NULL; + struct vrp_tree vrps = RB_INITIALIZER(&vrps); + struct brk_tree brks = RB_INITIALIZER(&brks); struct rusage ru; struct timeval start_time, now_time; @@ -625,7 +713,7 @@ main(int argc, char *argv[]) "proc exec unveil", NULL) == -1) err(1, "pledge"); - while ((c = getopt(argc, argv, "b:Bcd:e:jnorRs:t:T:vV")) != -1) + while ((c = getopt(argc, argv, "b:Bcd:e:f:jnorRs:t:T:vV")) != -1) switch (c) { case 'b': bind_addr = optarg; @@ -642,6 +730,10 @@ main(int argc, char *argv[]) case 'e': rsync_prog = optarg; break; + case 'f': + file = optarg; + noop = 1; + break; case 'j': outformats |= FORMAT_JSON; break; @@ -690,12 +782,6 @@ main(int argc, char *argv[]) signal(SIGPIPE, SIG_IGN); - if (timeout) { - signal(SIGALRM, suicide); - /* Commit suicide eventually - cron will normally start a new one */ - alarm(timeout); - } - if (cachedir == NULL) { warnx("cache directory required"); goto usage; @@ -705,16 +791,18 @@ main(int argc, char *argv[]) goto usage; } - if ((cachefd = open(cachedir, O_RDONLY, 0)) == -1) + if ((cachefd = open(cachedir, O_RDONLY | O_DIRECTORY)) == -1) err(1, "cache directory %s", cachedir); - if ((outdirfd = open(outputdir, O_RDONLY, 0)) == -1) + if ((outdirfd = open(outputdir, O_RDONLY | O_DIRECTORY)) == -1) err(1, "output directory %s", outputdir); + check_fs_size(cachefd, cachedir); + if (outformats == 0) outformats = FORMAT_OPENBGPD; if (talsz == 0) - talsz = tal_load_default(tals, TALSZ_MAX); + talsz = tal_load_default(); if (talsz == 0) err(1, "no TAL files found in %s", "/etc/rpki"); @@ -737,6 +825,9 @@ main(int argc, char *argv[]) if (fchdir(cachefd) == -1) err(1, "fchdir"); + if (timeout) + alarm(timeout); + /* Only allow access to the cache directory. */ if (unveil(".", "r") == -1) err(1, "%s: unveil", cachedir); @@ -771,6 +862,9 @@ main(int argc, char *argv[]) if (fchdir(cachefd) == -1) err(1, "fchdir"); + if (timeout) + alarm(timeout); + if (pledge("stdio rpath proc exec unveil", NULL) == -1) err(1, "pledge"); @@ -807,6 +901,9 @@ main(int argc, char *argv[]) if (fchdir(cachefd) == -1) err(1, "fchdir"); + if (timeout) + alarm(timeout); + if (pledge("stdio rpath inet dns recvfd", NULL) == -1) err(1, "pledge"); @@ -844,6 +941,9 @@ main(int argc, char *argv[]) if (fchdir(cachefd) == -1) err(1, "fchdir"); + if (timeout) + alarm(timeout); + if (pledge("stdio recvfd", NULL) == -1) err(1, "pledge"); @@ -853,8 +953,19 @@ main(int argc, char *argv[]) close(fd[0]); rrdp = fd[1]; - } else + } else { rrdp = -1; + rrdppid = -1; + } + + if (timeout) { + /* + * Commit suicide eventually + * cron will normally start a new one + */ + alarm(timeout); + signal(SIGALRM, suicide); + } /* TODO unveil cachedir and outputdir, no other access allowed */ if (pledge("stdio rpath wpath cpath fattr sendfd", NULL) == -1) @@ -891,53 +1002,55 @@ main(int argc, char *argv[]) */ for (i = 0; i < talsz; i++) - queue_add_tal(tals[i]); + queue_add_tal(tals[i], i); /* change working directory to the cache directory */ if (fchdir(cachefd) == -1) err(1, "fchdir"); while (entity_queue > 0 && !killme) { + int polltim; + for (i = 0; i < NPFD; i++) { pfd[i].events = POLLIN; if (queues[i]->queued) pfd[i].events |= POLLOUT; } - if ((c = poll(pfd, NPFD, INFTIM)) == -1) { + polltim = repo_next_timeout(INFTIM); + + if ((c = poll(pfd, NPFD, polltim)) == -1) { if (errno == EINTR) continue; err(1, "poll"); } for (i = 0; i < NPFD; i++) { - if (pfd[i].revents & (POLLERR|POLLNVAL)) - errx(1, "poll[%zu]: bad fd", i); - if (pfd[i].revents & POLLHUP) { - warnx("poll[%zu]: hangup", i); + if (pfd[i].revents & (POLLERR|POLLNVAL)) { + warnx("poll[%zu]: bad fd", i); hangup = 1; } + if (pfd[i].revents & POLLHUP) + hangup = 1; if (pfd[i].revents & POLLOUT) { - /* - * XXX work around deadlocks because of - * blocking read vs non-blocking writes. - */ - if (i > 1) - io_socket_nonblocking(pfd[i].fd); switch (msgbuf_write(queues[i])) { case 0: - errx(1, "write[%zu]: " + warnx("write[%zu]: " "connection closed", i); + hangup = 1; + break; case -1: - err(1, "write[%zu]", i); + warn("write[%zu]", i); + hangup = 1; + break; } - if (i > 1) - io_socket_blocking(pfd[i].fd); } } if (hangup) break; + repo_check_timeout(); + /* * Check the rsync and http process. * This means that one of our modules has completed @@ -946,72 +1059,38 @@ main(int argc, char *argv[]) */ if ((pfd[1].revents & POLLIN)) { - io_simple_read(rsync, &id, sizeof(id)); - io_simple_read(rsync, &ok, sizeof(ok)); - rsync_finish(id, ok); + b = io_buf_read(rsync, &rsyncbuf); + if (b != NULL) { + io_read_buf(b, &id, sizeof(id)); + io_read_buf(b, &ok, sizeof(ok)); + rsync_finish(id, ok); + ibuf_free(b); + } } if ((pfd[2].revents & POLLIN)) { - enum http_result res; - char *last_mod; - - io_simple_read(http, &id, sizeof(id)); - io_simple_read(http, &res, sizeof(res)); - io_str_read(http, &last_mod); - http_finish(id, res, last_mod); - free(last_mod); + b = io_buf_read(http, &httpbuf); + if (b != NULL) { + enum http_result res; + char *last_mod; + + io_read_buf(b, &id, sizeof(id)); + io_read_buf(b, &res, sizeof(res)); + io_read_str(b, &last_mod); + http_finish(id, res, last_mod); + free(last_mod); + ibuf_free(b); + } } /* * Handle RRDP requests here. */ if ((pfd[3].revents & POLLIN)) { - enum rrdp_msg type; - enum publish_type pt; - struct rrdp_session s; - char *uri, *last_mod, *data; - char hash[SHA256_DIGEST_LENGTH]; - size_t dsz; - - io_simple_read(rrdp, &type, sizeof(type)); - io_simple_read(rrdp, &id, sizeof(id)); - - switch (type) { - case RRDP_END: - io_simple_read(rrdp, &ok, sizeof(ok)); - rrdp_finish(id, ok); - break; - case RRDP_HTTP_REQ: - io_str_read(rrdp, &uri); - io_str_read(rrdp, &last_mod); - rrdp_http_fetch(id, uri, last_mod); - break; - case RRDP_SESSION: - io_str_read(rrdp, &s.session_id); - io_simple_read(rrdp, &s.serial, - sizeof(s.serial)); - io_str_read(rrdp, &s.last_mod); - rrdp_save_state(id, &s); - free(s.session_id); - free(s.last_mod); - break; - case RRDP_FILE: - io_simple_read(rrdp, &pt, sizeof(pt)); - if (pt != PUB_ADD) - io_simple_read(rrdp, &hash, - sizeof(hash)); - io_str_read(rrdp, &uri); - io_buf_read_alloc(rrdp, (void **)&data, &dsz); - - ok = rrdp_handle_file(id, pt, uri, - hash, sizeof(hash), data, dsz); - rrdp_file_resp(id, ok); - - free(uri); - free(data); - break; - default: - errx(1, "unexpected rrdp response"); + b = io_buf_read(rrdp, &rrdpbuf); + if (b != NULL) { + rrdp_process(b); + ibuf_free(b); } } @@ -1021,10 +1100,15 @@ main(int argc, char *argv[]) */ if ((pfd[0].revents & POLLIN)) { - entity_process(proc, &stats, &v); + b = io_buf_read(proc, &procbuf); + if (b != NULL) { + entity_process(b, &stats, &vrps, &brks); + ibuf_free(b); + } } } + signal(SIGALRM, SIG_DFL); if (killme) { syslog(LOG_CRIT|LOG_DAEMON, "excessive runtime (%d seconds), giving up", timeout); @@ -1075,7 +1159,7 @@ main(int argc, char *argv[]) /* processing did not finish because of error */ if (entity_queue != 0) - return 1; + errx(1, "not all files processed, giving up"); logx("all files parsed: generating output"); @@ -1096,15 +1180,21 @@ main(int argc, char *argv[]) if (fchdir(outdirfd) == -1) err(1, "fchdir output dir"); - if (outputfiles(&v, &stats)) + if (outputfiles(&vrps, &brks, &stats)) rc = 1; - + logx("Processing time %lld seconds " + "(%lld seconds user, %lld seconds system)", + (long long)stats.elapsed_time.tv_sec, + (long long)stats.user_time.tv_sec, + (long long)stats.system_time.tv_sec); logx("Route Origin Authorizations: %zu (%zu failed parse, %zu invalid)", stats.roas, stats.roas_fail, stats.roas_invalid); - logx("Certificates: %zu (%zu failed parse, %zu invalid)", - stats.certs, stats.certs_fail, stats.certs_invalid); - logx("Trust Anchor Locators: %zu", stats.tals); + logx("BGPsec Router Certificates: %zu", stats.brks); + logx("Certificates: %zu (%zu invalid)", + stats.certs, stats.certs_fail); + logx("Trust Anchor Locators: %zu (%zu invalid)", + stats.tals, talsz - stats.tals); logx("Manifests: %zu (%zu failed parse, %zu stale)", stats.mfts, stats.mfts_fail, stats.mfts_stale); logx("Certificate revocation lists: %zu", stats.crls); @@ -1116,10 +1206,6 @@ main(int argc, char *argv[]) /* Memory cleanup. */ repo_free(); - - for (i = 0; i < outsz; i++) - roa_free(out[i]); - free(out); return rc; Index: usr.sbin/rpki-client/mft.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/mft.c,v retrieving revision 1.32 diff -u -p -u -r1.32 mft.c --- usr.sbin/rpki-client/mft.c 29 Mar 2021 06:50:44 -0000 1.32 +++ usr.sbin/rpki-client/mft.c 6 Nov 2021 18:17:50 -0000 @@ -40,6 +40,8 @@ struct parse { struct mft *res; /* result object */ }; +static ASN1_OBJECT *mft_oid; + static const char * gentime2str(const ASN1_GENERALIZEDTIME *time) { @@ -70,6 +72,7 @@ generalizedtime_to_tm(const ASN1_GENERAL data = ASN1_STRING_get0_data(gtime); len = ASN1_STRING_length(gtime); + memset(tm, 0, sizeof(*tm)); return ASN1_time_parse(data, len, tm, V_ASN1_GENERALIZEDTIME) == V_ASN1_GENERALIZEDTIME; } @@ -129,7 +132,7 @@ mft_parse_filehash(struct parse *p, cons const ASN1_TYPE *file, *hash; char *fn = NULL; const unsigned char *d = os->data; - size_t dsz = os->length, sz; + size_t dsz = os->length; int rc = 0; struct mftfile *fent; @@ -169,7 +172,7 @@ mft_parse_filehash(struct parse *p, cons warnx("%s: path components disallowed in filename: %s", p->fn, fn); goto out; - } else if ((sz = strlen(fn)) <= 4) { + } else if (strlen(fn) <= 4) { warnx("%s: filename must be large enough for suffix part: %s", p->fn, fn); goto out; @@ -193,12 +196,6 @@ mft_parse_filehash(struct parse *p, cons } /* Insert the filename and hash value. */ - - p->res->files = recallocarray(p->res->files, p->res->filesz, - p->res->filesz + 1, sizeof(struct mftfile)); - if (p->res->files == NULL) - err(1, NULL); - fent = &p->res->files[p->res->filesz++]; fent->file = fn; @@ -231,6 +228,16 @@ mft_parse_flist(struct parse *p, const A goto out; } + if (sk_ASN1_TYPE_num(seq) > MAX_MANIFEST_ENTRIES) { + warnx("%s: %d exceeds manifest entry limit (%d)", p->fn, + sk_ASN1_TYPE_num(seq), MAX_MANIFEST_ENTRIES); + goto out; + } + + p->res->files = calloc(sk_ASN1_TYPE_num(seq), sizeof(struct mftfile)); + if (p->res->files == NULL) + err(1, NULL); + for (i = 0; i < sk_ASN1_TYPE_num(seq); i++) { t = sk_ASN1_TYPE_value(seq, i); if (t->type != V_ASN1_SEQUENCE) { @@ -243,7 +250,7 @@ mft_parse_flist(struct parse *p, const A } rc = 1; -out: + out: sk_ASN1_TYPE_pop_free(seq, ASN1_TYPE_free); return rc; } @@ -258,9 +265,9 @@ mft_parse_econtent(const unsigned char * ASN1_SEQUENCE_ANY *seq; const ASN1_TYPE *t; const ASN1_GENERALIZEDTIME *from, *until; - BIGNUM *mft_seqnum = NULL; long mft_version; - int i, rc = -1; + BIGNUM *mft_seqnum = NULL; + int i = 0, rc = -1; if ((seq = d2i_ASN1_SEQUENCE_ANY(NULL, &d, dsz)) == NULL) { cryptowarnx("%s: RFC 6486 section 4.2: Manifest: " @@ -268,8 +275,7 @@ mft_parse_econtent(const unsigned char * goto out; } - /* The profile version is optional. */ - + /* Test if the optional profile version field is present. */ if (sk_ASN1_TYPE_num(seq) != 5 && sk_ASN1_TYPE_num(seq) != 6) { warnx("%s: RFC 6486 section 4.2: Manifest: " @@ -278,25 +284,22 @@ mft_parse_econtent(const unsigned char * goto out; } - /* Start with optional profile version. */ - - i = 0; + /* Parse the optional version field */ if (sk_ASN1_TYPE_num(seq) == 6) { t = sk_ASN1_TYPE_value(seq, i++); - if (t->type != V_ASN1_INTEGER) { - warnx("%s: RFC 6486 section 4.2.1: version: " - "want ASN.1 integer, have %s (NID %d)", - p->fn, ASN1_tag2str(t->type), t->type); - goto out; - } + d = t->value.asn1_string->data; + dsz = t->value.asn1_string->length; - if (t->value.integer == NULL) + if (cms_econtent_version(p->fn, &d, dsz, &mft_version) == -1) goto out; - mft_version = ASN1_INTEGER_get(t->value.integer); - if (mft_version != 0) { - warnx("%s: RFC 6486 section 4.2.1: version: " - "want 0, have %ld", p->fn, mft_version); + switch (mft_version) { + case 0: + warnx("%s: incorrect encoding for version 0", p->fn); + goto out; + default: + warnx("%s: version %ld not supported (yet)", p->fn, + mft_version); goto out; } } @@ -412,7 +415,7 @@ out: * The MFT content is otherwise returned. */ struct mft * -mft_parse(X509 **x509, const char *fn) +mft_parse(X509 **x509, const char *fn, const unsigned char *der, size_t len) { struct parse p; int c, rc = 0; @@ -422,8 +425,14 @@ mft_parse(X509 **x509, const char *fn) memset(&p, 0, sizeof(struct parse)); p.fn = fn; - cms = cms_parse_validate(x509, fn, "1.2.840.113549.1.9.16.1.26", - &cmsz); + if (mft_oid == NULL) { + mft_oid = OBJ_txt2obj("1.2.840.113549.1.9.16.1.26", 1); + if (mft_oid == NULL) + errx(1, "OBJ_txt2obj for %s failed", + "1.2.840.113549.1.9.16.1.26"); + } + + cms = cms_parse_validate(x509, fn, der, len, mft_oid, &cmsz); if (cms == NULL) return NULL; assert(*x509 != NULL); @@ -486,7 +495,7 @@ mft_check(const char *fn, struct mft *p) { size_t i; int rc = 1; - char *cp, *path = NULL; + char *cp, *h, *path = NULL; /* Check hash of file now, but first build path for it */ cp = strrchr(fn, '/'); @@ -495,6 +504,13 @@ mft_check(const char *fn, struct mft *p) for (i = 0; i < p->filesz; i++) { const struct mftfile *m = &p->files[i]; + if (!valid_filename(m->file)) { + if (base64_encode(m->hash, sizeof(m->hash), &h) == -1) + errx(1, "base64_encode failed in %s", __func__); + warnx("%s: unsupported filename for %s", fn, h); + free(h); + continue; + } if (asprintf(&path, "%.*s/%s", (int)(cp - fn), fn, m->file) == -1) err(1, NULL); @@ -561,7 +577,7 @@ mft_buffer(struct ibuf *b, const struct * Result must be passed to mft_free(). */ struct mft * -mft_read(int fd) +mft_read(struct ibuf *b) { struct mft *p = NULL; size_t i; @@ -569,22 +585,22 @@ mft_read(int fd) if ((p = calloc(1, sizeof(struct mft))) == NULL) err(1, NULL); - io_simple_read(fd, &p->stale, sizeof(int)); - io_str_read(fd, &p->file); - assert(p->file); - io_simple_read(fd, &p->filesz, sizeof(size_t)); + io_read_buf(b, &p->stale, sizeof(int)); + io_read_str(b, &p->file); + io_read_buf(b, &p->filesz, sizeof(size_t)); + assert(p->file); if ((p->files = calloc(p->filesz, sizeof(struct mftfile))) == NULL) err(1, NULL); for (i = 0; i < p->filesz; i++) { - io_str_read(fd, &p->files[i].file); - io_simple_read(fd, p->files[i].hash, SHA256_DIGEST_LENGTH); + io_read_str(b, &p->files[i].file); + io_read_buf(b, p->files[i].hash, SHA256_DIGEST_LENGTH); } - io_str_read(fd, &p->aia); - io_str_read(fd, &p->aki); - io_str_read(fd, &p->ski); + io_read_str(b, &p->aia); + io_read_str(b, &p->aki); + io_read_str(b, &p->ski); assert(p->aia && p->aki && p->ski); return p; Index: usr.sbin/rpki-client/mkdir.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/mkdir.c,v retrieving revision 1.6 diff -u -p -u -r1.6 mkdir.c --- usr.sbin/rpki-client/mkdir.c 29 Mar 2021 04:01:17 -0000 1.6 +++ usr.sbin/rpki-client/mkdir.c 6 Nov 2021 18:18:06 -0000 @@ -39,9 +39,7 @@ /* * mkpath -- create directories. - * path - path - * mode - file mode of terminal directory - * dir_mode - file mode of intermediate directories + * dir - path to create directories for */ int mkpath(const char *dir) @@ -53,6 +51,7 @@ mkpath(const char *dir) return -1; slash = path; + for (;;) { slash += strspn(slash, "/"); slash += strcspn(slash, "/"); Index: usr.sbin/rpki-client/output-bgpd.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/output-bgpd.c,v retrieving revision 1.20 diff -u -p -u -r1.20 output-bgpd.c --- usr.sbin/rpki-client/output-bgpd.c 29 Mar 2021 03:39:14 -0000 1.20 +++ usr.sbin/rpki-client/output-bgpd.c 6 Nov 2021 18:18:39 -0000 @@ -20,9 +20,9 @@ #include "extern.h" int -output_bgpd(FILE *out, struct vrp_tree *vrps, struct stats *st) +output_bgpd(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct stats *st) { - char ipbuf[64], maxlenbuf[100]; struct vrp *v; if (outputheader(out, st) < 0) @@ -32,6 +32,8 @@ output_bgpd(FILE *out, struct vrp_tree * return -1; RB_FOREACH(v, vrp_tree, vrps) { + char ipbuf[64], maxlenbuf[100]; + ip_addr_print(&v->addr, v->afi, ipbuf, sizeof(ipbuf)); if (v->maxlength > v->addr.prefixlen) { int ret = snprintf(maxlenbuf, sizeof(maxlenbuf), @@ -40,8 +42,8 @@ output_bgpd(FILE *out, struct vrp_tree * return -1; } else maxlenbuf[0] = '\0'; - if (fprintf(out, "\t%s %ssource-as %u\n", - ipbuf, maxlenbuf, v->asid) < 0) + if (fprintf(out, "\t%s %ssource-as %u expires %lld\n", + ipbuf, maxlenbuf, v->asid, (long long)v->expires) < 0) return -1; } Index: usr.sbin/rpki-client/output-bird.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/output-bird.c,v retrieving revision 1.10 diff -u -p -u -r1.10 output-bird.c --- usr.sbin/rpki-client/output-bird.c 12 Sep 2020 15:46:48 -0000 1.10 +++ usr.sbin/rpki-client/output-bird.c 6 Nov 2021 18:22:31 -0000 @@ -21,10 +21,10 @@ #include "extern.h" int -output_bird1v4(FILE *out, struct vrp_tree *vrps, struct stats *st) +output_bird1v4(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct stats *st) { extern const char *bird_tablename; - char buf[64]; struct vrp *v; if (outputheader(out, st) < 0) @@ -34,6 +34,8 @@ output_bird1v4(FILE *out, struct vrp_tre return -1; RB_FOREACH(v, vrp_tree, vrps) { + char buf[64]; + if (v->afi == AFI_IPV4) { ip_addr_print(&v->addr, v->afi, buf, sizeof(buf)); if (fprintf(out, "\troa %s max %u as %u;\n", buf, @@ -48,10 +50,10 @@ output_bird1v4(FILE *out, struct vrp_tre } int -output_bird1v6(FILE *out, struct vrp_tree *vrps, struct stats *st) +output_bird1v6(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct stats *st) { extern const char *bird_tablename; - char buf[64]; struct vrp *v; if (outputheader(out, st) < 0) @@ -61,6 +63,8 @@ output_bird1v6(FILE *out, struct vrp_tre return -1; RB_FOREACH(v, vrp_tree, vrps) { + char buf[64]; + if (v->afi == AFI_IPV6) { ip_addr_print(&v->addr, v->afi, buf, sizeof(buf)); if (fprintf(out, "\troa %s max %u as %u;\n", buf, @@ -75,10 +79,10 @@ output_bird1v6(FILE *out, struct vrp_tre } int -output_bird2(FILE *out, struct vrp_tree *vrps, struct stats *st) +output_bird2(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct stats *st) { extern const char *bird_tablename; - char buf[64]; struct vrp *v; time_t now = time(NULL); @@ -93,6 +97,8 @@ output_bird2(FILE *out, struct vrp_tree return -1; RB_FOREACH(v, vrp_tree, vrps) { + char buf[64]; + if (v->afi == AFI_IPV4) { ip_addr_print(&v->addr, v->afi, buf, sizeof(buf)); if (fprintf(out, "\troute %s max %u as %u;\n", buf, @@ -106,6 +112,8 @@ output_bird2(FILE *out, struct vrp_tree return -1; RB_FOREACH(v, vrp_tree, vrps) { + char buf[64]; + if (v->afi == AFI_IPV6) { ip_addr_print(&v->addr, v->afi, buf, sizeof(buf)); if (fprintf(out, "\troute %s max %u as %u;\n", buf, Index: usr.sbin/rpki-client/output-csv.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/output-csv.c,v retrieving revision 1.8 diff -u -p -u -r1.8 output-csv.c --- usr.sbin/rpki-client/output-csv.c 12 Sep 2020 15:46:48 -0000 1.8 +++ usr.sbin/rpki-client/output-csv.c 6 Nov 2021 18:23:52 -0000 @@ -20,18 +20,22 @@ #include "extern.h" int -output_csv(FILE *out, struct vrp_tree *vrps, struct stats *st) +output_csv(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct stats *st) { - char buf[64]; struct vrp *v; - if (fprintf(out, "ASN,IP Prefix,Max Length,Trust Anchor\n") < 0) + if (fprintf(out, "ASN,IP Prefix,Max Length,Trust Anchor,Expires\n") < 0) return -1; RB_FOREACH(v, vrp_tree, vrps) { + char buf[64]; + ip_addr_print(&v->addr, v->afi, buf, sizeof(buf)); - if (fprintf(out, "AS%u,%s,%u,%s\n", v->asid, buf, v->maxlength, - v->tal) < 0) + + if (fprintf(out, "AS%u,%s,%u,%s,%lld\n", v->asid, buf, + v->maxlength, taldescs[v->talid], + (long long)v->expires) < 0) return -1; } return 0; Index: usr.sbin/rpki-client/output-json.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/output-json.c,v retrieving revision 1.15 diff -u -p -u -r1.15 output-json.c --- usr.sbin/rpki-client/output-json.c 8 Apr 2021 19:49:27 -0000 1.15 +++ usr.sbin/rpki-client/output-json.c 6 Nov 2021 18:23:32 -0000 @@ -28,6 +28,7 @@ outputheader_json(FILE *out, struct stat char hn[NI_MAXHOST], tbuf[26]; struct tm *tp; time_t t; + size_t i; time(&t); setenv("TZ", "UTC", 1); @@ -46,11 +47,28 @@ outputheader_json(FILE *out, struct stat "\t\t\"roas\": %zu,\n" "\t\t\"failedroas\": %zu,\n" "\t\t\"invalidroas\": %zu,\n" + "\t\t\"bgpsec_pubkeys\": %zu,\n" "\t\t\"certificates\": %zu,\n" - "\t\t\"failcertificates\": %zu,\n" "\t\t\"invalidcertificates\": %zu,\n" "\t\t\"tals\": %zu,\n" - "\t\t\"talfiles\": \"%s\",\n" + "\t\t\"invalidtals\": %zu,\n" + "\t\t\"talfiles\": [\n", + hn, tbuf, (long long)st->elapsed_time.tv_sec, + (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, + st->roas, st->roas_fail, st->roas_invalid, + st->brks, st->certs, st->certs_fail, + st->tals, talsz - st->tals) < 0) + return -1; + + for (i = 0; i < talsz; i++) { + if (fprintf(out, + "\t\t\t\"%s\"%s\n", + tals[i], i == talsz - 1 ? "" : ",") < 0) + return -1; + } + + if (fprintf(out, + "\t\t],\n" "\t\t\"manifests\": %zu,\n" "\t\t\"failedmanifests\": %zu,\n" "\t\t\"stalemanifests\": %zu,\n" @@ -62,11 +80,6 @@ outputheader_json(FILE *out, struct stat "\t\t\"cachedir_del_files\": %zu,\n" "\t\t\"cachedir_del_dirs\": %zu\n" "\t},\n\n", - hn, tbuf, (long long)st->elapsed_time.tv_sec, - (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->roas, st->roas_fail, st->roas_invalid, - st->certs, st->certs_fail, st->certs_invalid, - st->tals, st->talnames, st->mfts, st->mfts_fail, st->mfts_stale, st->crls, st->gbrs, @@ -78,10 +91,12 @@ outputheader_json(FILE *out, struct stat } int -output_json(FILE *out, struct vrp_tree *vrps, struct stats *st) +output_json(FILE *out, struct vrp_tree *vrps, struct brk_tree *brks, + struct stats *st) { char buf[64]; struct vrp *v; + struct brk *b; int first = 1; if (outputheader_json(out, st) < 0) @@ -91,18 +106,37 @@ output_json(FILE *out, struct vrp_tree * return -1; RB_FOREACH(v, vrp_tree, vrps) { - if (first) - first = 0; - else { + if (!first) { if (fprintf(out, ",\n") < 0) return -1; } + first = 0; ip_addr_print(&v->addr, v->afi, buf, sizeof(buf)); - if (fprintf(out, "\t\t{ \"asn\": \"AS%u\", \"prefix\": \"%s\", " - "\"maxLength\": %u, \"ta\": \"%s\" }", - v->asid, buf, v->maxlength, v->tal) < 0) + if (fprintf(out, "\t\t{ \"asn\": %u, \"prefix\": \"%s\", " + "\"maxLength\": %u, \"ta\": \"%s\", \"expires\": %lld }", + v->asid, buf, v->maxlength, taldescs[v->talid], + (long long)v->expires) + < 0) + return -1; + } + + if (fprintf(out, "\n\t],\n\n\t\"bgpsec_keys\": [\n") < 0) + return -1; + + first = 1; + RB_FOREACH(b, brk_tree, brks) { + if (!first) { + if (fprintf(out, ",\n") < 0) + return -1; + } + first = 0; + + if (fprintf(out, "\t\t{ \"asn\": %u, \"ski\": \"%s\", " + "\"pubkey\": \"%s\", \"ta\": \"%s\", \"expires\": %lld }", + b->asid, b->ski, b->pubkey, taldescs[b->talid], + (long long)b->expires) < 0) return -1; } Index: usr.sbin/rpki-client/output.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/output.c,v retrieving revision 1.21 diff -u -p -u -r1.21 output.c --- usr.sbin/rpki-client/output.c 2 Mar 2021 09:08:59 -0000 1.21 +++ usr.sbin/rpki-client/output.c 6 Nov 2021 18:23:13 -0000 @@ -64,7 +64,8 @@ static char output_name[PATH_MAX]; static const struct outputs { int format; char *name; - int (*fn)(FILE *, struct vrp_tree *, struct stats *); + int (*fn)(FILE *, struct vrp_tree *, struct brk_tree *, + struct stats *); } outputs[] = { { FORMAT_OPENBGPD, "openbgpd", output_bgpd }, { FORMAT_BIRD, "bird1v4", output_bird1v4 }, @@ -82,7 +83,7 @@ static void sig_handler(int); static void set_signal_handler(void); int -outputfiles(struct vrp_tree *v, struct stats *st) +outputfiles(struct vrp_tree *v, struct brk_tree *b, struct stats *st) { int i, rc = 0; @@ -101,7 +102,7 @@ outputfiles(struct vrp_tree *v, struct s rc = 1; continue; } - if ((*outputs[i].fn)(fout, v, st) != 0) { + if ((*outputs[i].fn)(fout, v, b, st) != 0) { warn("output for %s format failed", outputs[i].name); fclose(fout); output_cleantmp(); @@ -200,6 +201,7 @@ outputheader(FILE *out, struct stats *st char hn[NI_MAXHOST], tbuf[80]; struct tm *tp; time_t t; + size_t i; time(&t); setenv("TZ", "UTC", 1); @@ -210,20 +212,31 @@ outputheader(FILE *out, struct stats *st if (fprintf(out, "# Generated on host %s at %s\n" - "# Processing time %lld seconds (%lld seconds user, %lld seconds system)\n" + "# Processing time %lld seconds (%llds user, %llds system)\n" "# Route Origin Authorizations: %zu (%zu failed parse, %zu invalid)\n" - "# Certificates: %zu (%zu failed parse, %zu invalid)\n" - "# Trust Anchor Locators: %zu (%s)\n" + "# BGPsec Router Certificates: %zu\n" + "# Certificates: %zu (%zu invalid)\n", + hn, tbuf, (long long)st->elapsed_time.tv_sec, + (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, + st->roas, st->roas_fail, st->roas_invalid, + st->brks, st->certs, st->certs_fail) < 0) + return -1; + + if (fprintf(out, + "# Trust Anchor Locators: %zu (%zu invalid) [", st->tals, + talsz - st->tals) < 0) + return -1; + for (i = 0; i < talsz; i++) + if (fprintf(out, " %s", tals[i]) < 0) + return -1; + + if (fprintf(out, + " ]\n" "# Manifests: %zu (%zu failed parse, %zu stale)\n" "# Certificate revocation lists: %zu\n" "# Ghostbuster records: %zu\n" "# Repositories: %zu\n" "# VRP Entries: %zu (%zu unique)\n", - hn, tbuf, (long long)st->elapsed_time.tv_sec, - (long long)st->user_time.tv_sec, (long long)st->system_time.tv_sec, - st->roas, st->roas_fail, st->roas_invalid, - st->certs, st->certs_fail, st->certs_invalid, - st->tals, st->talnames, st->mfts, st->mfts_fail, st->mfts_stale, st->crls, st->gbrs, Index: usr.sbin/rpki-client/parser.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/parser.c,v retrieving revision 1.7 diff -u -p -u -r1.7 parser.c --- usr.sbin/rpki-client/parser.c 1 Apr 2021 08:29:10 -0000 1.7 +++ usr.sbin/rpki-client/parser.c 8 Nov 2021 13:32:34 -0000 @@ -37,18 +37,22 @@ #include "extern.h" -static void build_chain(const struct auth *, STACK_OF(X509) **); -static void build_crls(const struct auth *, struct crl_tree *, - STACK_OF(X509_CRL) **); +static void build_chain(const struct auth *, STACK_OF(X509) **); +static struct crl *get_crl(const struct auth *); +static void build_crls(const struct crl *, STACK_OF(X509_CRL) **); + +static X509_STORE_CTX *ctx; +static X509_STORE *store; +static struct auth_tree auths = RB_INITIALIZER(&auths); +static struct crl_tree crlt = RB_INITIALIZER(&crlt); + /* * Parse and validate a ROA. * This is standard stuff. * Returns the roa on success, NULL on failure. */ static struct roa * -proc_parser_roa(struct entity *entp, - X509_STORE *store, X509_STORE_CTX *ctx, - struct auth_tree *auths, struct crl_tree *crlt) +proc_parser_roa(struct entity *entp, const unsigned char *der, size_t len) { struct roa *roa; X509 *x509; @@ -56,20 +60,23 @@ proc_parser_roa(struct entity *entp, struct auth *a; STACK_OF(X509) *chain; STACK_OF(X509_CRL) *crls; + struct crl *crl; - if ((roa = roa_parse(&x509, entp->file)) == NULL) + if ((roa = roa_parse(&x509, entp->file, der, len)) == NULL) return NULL; - a = valid_ski_aki(entp->file, auths, roa->ski, roa->aki); - + a = valid_ski_aki(entp->file, &auths, roa->ski, roa->aki); build_chain(a, &chain); - build_crls(a, crlt, &crls); + crl = get_crl(a); + build_crls(crl, &crls); assert(x509 != NULL); - if (!X509_STORE_CTX_init(ctx, store, x509, chain)) + if (!X509_STORE_CTX_init(ctx, store, x509, NULL)) cryptoerrx("X509_STORE_CTX_init"); X509_STORE_CTX_set_flags(ctx, X509_V_FLAG_IGNORE_CRITICAL | X509_V_FLAG_CRL_CHECK); + X509_STORE_CTX_set_depth(ctx, MAX_CERT_DEPTH); + X509_STORE_CTX_set0_trusted_stack(ctx, chain); X509_STORE_CTX_set0_crls(ctx, crls); if (X509_verify_cert(ctx) <= 0) { @@ -85,18 +92,34 @@ proc_parser_roa(struct entity *entp, return NULL; } X509_STORE_CTX_cleanup(ctx); - sk_X509_free(chain); - sk_X509_CRL_free(crls); - X509_free(x509); + + /* + * Check CRL to figure out the soonest transitive expiry moment + */ + if (crl != NULL && roa->expires > crl->expires) + roa->expires = crl->expires; + + /* + * Scan the cert tree to figure out the soonest transitive + * expiry moment + */ + for (; a != NULL; a = a->parent) { + if (roa->expires > a->cert->expires) + roa->expires = a->cert->expires; + } /* * If the ROA isn't valid, we accept it anyway and depend upon * the code around roa_read() to check the "valid" field itself. */ - if (valid_roa(entp->file, auths, roa)) + if (valid_roa(entp->file, &auths, roa)) roa->valid = 1; + sk_X509_free(chain); + sk_X509_CRL_free(crls); + X509_free(x509); + return roa; } @@ -111,8 +134,7 @@ proc_parser_roa(struct entity *entp, * Return the mft on success or NULL on failure. */ static struct mft * -proc_parser_mft(struct entity *entp, X509_STORE *store, X509_STORE_CTX *ctx, - struct auth_tree *auths, struct crl_tree *crlt) +proc_parser_mft(struct entity *entp, const unsigned char *der, size_t len) { struct mft *mft; X509 *x509; @@ -120,17 +142,19 @@ proc_parser_mft(struct entity *entp, X50 struct auth *a; STACK_OF(X509) *chain; - if ((mft = mft_parse(&x509, entp->file)) == NULL) + if ((mft = mft_parse(&x509, entp->file, der, len)) == NULL) return NULL; - a = valid_ski_aki(entp->file, auths, mft->ski, mft->aki); + a = valid_ski_aki(entp->file, &auths, mft->ski, mft->aki); build_chain(a, &chain); - if (!X509_STORE_CTX_init(ctx, store, x509, chain)) + if (!X509_STORE_CTX_init(ctx, store, x509, NULL)) cryptoerrx("X509_STORE_CTX_init"); /* CRL checked disabled here because CRL is referenced from mft */ X509_STORE_CTX_set_flags(ctx, X509_V_FLAG_IGNORE_CRITICAL); + X509_STORE_CTX_set_depth(ctx, MAX_CERT_DEPTH); + X509_STORE_CTX_set0_trusted_stack(ctx, chain); if (X509_verify_cert(ctx) <= 0) { c = X509_STORE_CTX_get_error(ctx); @@ -162,41 +186,36 @@ proc_parser_mft(struct entity *entp, X50 * parse failure. */ static struct cert * -proc_parser_cert(const struct entity *entp, - X509_STORE *store, X509_STORE_CTX *ctx, - struct auth_tree *auths, struct crl_tree *crlt) +proc_parser_cert(const struct entity *entp, const unsigned char *der, + size_t len) { struct cert *cert; X509 *x509; int c; - struct auth *a = NULL, *na; - char *tal; + struct auth *a = NULL; STACK_OF(X509) *chain; STACK_OF(X509_CRL) *crls; - assert(!entp->has_pkey); + assert(!entp->has_data); /* Extract certificate data and X509. */ - cert = cert_parse(&x509, entp->file); + cert = cert_parse(&x509, entp->file, der, len); if (cert == NULL) return NULL; - a = valid_ski_aki(entp->file, auths, cert->ski, cert->aki); + a = valid_ski_aki(entp->file, &auths, cert->ski, cert->aki); build_chain(a, &chain); - build_crls(a, crlt, &crls); - - /* - * Validate certificate chain w/CRLs. - * Only check the CRLs if specifically asked. - */ + build_crls(get_crl(a), &crls); assert(x509 != NULL); - if (!X509_STORE_CTX_init(ctx, store, x509, chain)) + if (!X509_STORE_CTX_init(ctx, store, x509, NULL)) cryptoerrx("X509_STORE_CTX_init"); X509_STORE_CTX_set_flags(ctx, X509_V_FLAG_IGNORE_CRITICAL | X509_V_FLAG_CRL_CHECK); + X509_STORE_CTX_set_depth(ctx, MAX_CERT_DEPTH); + X509_STORE_CTX_set0_trusted_stack(ctx, chain); X509_STORE_CTX_set0_crls(ctx, crls); if (X509_verify_cert(ctx) <= 0) { @@ -214,67 +233,53 @@ proc_parser_cert(const struct entity *en X509_STORE_CTX_cleanup(ctx); sk_X509_free(chain); sk_X509_CRL_free(crls); + X509_free(x509); + + cert->talid = a->cert->talid; /* Validate the cert to get the parent */ - if (!valid_cert(entp->file, auths, cert)) { - X509_free(x509); // needed? XXX - return cert; + if (!valid_cert(entp->file, &auths, cert)) { + cert_free(cert); + return NULL; } /* - * Add validated certs to the RPKI auth tree. + * Add validated CA certs to the RPKI auth tree. */ - - cert->valid = 1; - - na = malloc(sizeof(*na)); - if (na == NULL) - err(1, NULL); - - tal = a->tal; - - na->parent = a; - na->cert = cert; - na->tal = tal; - na->fn = strdup(entp->file); - if (na->fn == NULL) - err(1, NULL); - - if (RB_INSERT(auth_tree, auths, na) != NULL) - err(1, "auth tree corrupted"); + if (cert->purpose == CERT_PURPOSE_CA) { + if (!auth_insert(&auths, cert, a)) { + cert_free(cert); + return NULL; + } + } return cert; } - /* * Root certificates come from TALs (has a pkey and is self-signed). * Parse the certificate, ensure that it's public key matches the * known public key from the TAL, and then validate the RPKI - * content. If valid, we add it as a trusted root (trust anchor) to - * "store". + * content. * * This returns a certificate (which must not be freed) or NULL on * parse failure. */ static struct cert * -proc_parser_root_cert(const struct entity *entp, - X509_STORE *store, X509_STORE_CTX *ctx, - struct auth_tree *auths, struct crl_tree *crlt) +proc_parser_root_cert(const struct entity *entp, const unsigned char *der, + size_t len) { char subject[256]; ASN1_TIME *notBefore, *notAfter; X509_NAME *name; struct cert *cert; X509 *x509; - struct auth *na; - char *tal; - assert(entp->has_pkey); + assert(entp->has_data); /* Extract certificate data and X509. */ - cert = ta_parse(&x509, entp->file, entp->pkey, entp->pkeysz); + cert = ta_parse(&x509, entp->file, der, len, entp->data, entp->datasz); if (cert == NULL) return NULL; @@ -307,78 +312,88 @@ proc_parser_root_cert(const struct entit subject); goto badcert; } - if (!valid_ta(entp->file, auths, cert)) { + if (!valid_ta(entp->file, &auths, cert)) { warnx("%s: certificate not a valid ta, subject='%s'", entp->file, subject); goto badcert; } - /* - * Add valid roots to the RPKI auth tree and as a trusted root - * for chain validation to the X509_STORE. - */ - - cert->valid = 1; - - na = malloc(sizeof(*na)); - if (na == NULL) - err(1, NULL); - - if ((tal = strdup(entp->descr)) == NULL) - err(1, NULL); - - na->parent = NULL; - na->cert = cert; - na->tal = tal; - na->fn = strdup(entp->file); - if (na->fn == NULL) - err(1, NULL); + X509_free(x509); - if (RB_INSERT(auth_tree, auths, na) != NULL) - err(1, "auth tree corrupted"); + cert->talid = entp->talid; - X509_STORE_add_cert(store, x509); + /* + * Add valid roots to the RPKI auth tree. + */ + if (!auth_insert(&auths, cert, NULL)) { + cert_free(cert); + return NULL; + } return cert; + badcert: - X509_free(x509); // needed? XXX - return cert; + X509_free(x509); + cert_free(cert); + return NULL; } /* * Parse a certificate revocation list * This simply parses the CRL content itself, optionally validating it * within the digest if it comes from a manifest, then adds it to the - * store of CRLs. + * CRL tree. */ static void -proc_parser_crl(struct entity *entp, X509_STORE *store, - X509_STORE_CTX *ctx, struct crl_tree *crlt) +proc_parser_crl(struct entity *entp, const unsigned char *der, size_t len) { X509_CRL *x509_crl; struct crl *crl; + const ASN1_TIME *at; + struct tm expires_tm; - if ((x509_crl = crl_parse(entp->file)) != NULL) { + if ((x509_crl = crl_parse(entp->file, der, len)) != NULL) { if ((crl = malloc(sizeof(*crl))) == NULL) err(1, NULL); if ((crl->aki = x509_crl_get_aki(x509_crl, entp->file)) == - NULL) - errx(1, "x509_crl_get_aki failed"); + NULL) { + warnx("x509_crl_get_aki failed"); + goto err; + } + crl->x509_crl = x509_crl; - if (RB_INSERT(crl_tree, crlt, crl) != NULL) { + /* extract expire time for later use */ + at = X509_CRL_get0_nextUpdate(x509_crl); + if (at == NULL) { + warnx("%s: X509_CRL_get0_nextUpdate failed", + entp->file); + goto err; + } + memset(&expires_tm, 0, sizeof(expires_tm)); + if (ASN1_time_parse(at->data, at->length, &expires_tm, + 0) == -1) { + warnx("%s: ASN1_time_parse failed", entp->file); + goto err; + } + if ((crl->expires = mktime(&expires_tm)) == -1) + errx(1, "%s: mktime failed", entp->file); + + if (RB_INSERT(crl_tree, &crlt, crl) != NULL) { warnx("%s: duplicate AKI %s", entp->file, crl->aki); - free_crl(crl); + goto err; } } + return; + err: + free_crl(crl); } /* * Parse a ghostbuster record */ static void -proc_parser_gbr(struct entity *entp, X509_STORE *store, - X509_STORE_CTX *ctx, struct auth_tree *auths, struct crl_tree *crlt) +proc_parser_gbr(struct entity *entp, const unsigned char *der, size_t len) { struct gbr *gbr; X509 *x509; @@ -387,19 +402,21 @@ proc_parser_gbr(struct entity *entp, X50 STACK_OF(X509) *chain; STACK_OF(X509_CRL) *crls; - if ((gbr = gbr_parse(&x509, entp->file)) == NULL) + if ((gbr = gbr_parse(&x509, entp->file, der, len)) == NULL) return; - a = valid_ski_aki(entp->file, auths, gbr->ski, gbr->aki); + a = valid_ski_aki(entp->file, &auths, gbr->ski, gbr->aki); build_chain(a, &chain); - build_crls(a, crlt, &crls); + build_crls(get_crl(a), &crls); assert(x509 != NULL); - if (!X509_STORE_CTX_init(ctx, store, x509, chain)) + if (!X509_STORE_CTX_init(ctx, store, x509, NULL)) cryptoerrx("X509_STORE_CTX_init"); X509_STORE_CTX_set_flags(ctx, X509_V_FLAG_IGNORE_CRITICAL | X509_V_FLAG_CRL_CHECK); + X509_STORE_CTX_set_depth(ctx, MAX_CERT_DEPTH); + X509_STORE_CTX_set0_trusted_stack(ctx, chain); X509_STORE_CTX_set0_crls(ctx, crls); if (X509_verify_cert(ctx) <= 0) { @@ -417,9 +434,9 @@ proc_parser_gbr(struct entity *entp, X50 } /* - * Use the parent to walk the tree to the root and build a certificate - * chain from cert->x509. Do not include the root node since this node - * should already be in the X509_STORE as a trust anchor. + * Walk the certificate tree to the root and build a certificate + * chain from cert->x509. All certs in the tree are validated and + * can be loaded as trusted stack into the validator. */ static void build_chain(const struct auth *a, STACK_OF(X509) **chain) @@ -431,7 +448,7 @@ build_chain(const struct auth *a, STACK_ if ((*chain = sk_X509_new_null()) == NULL) err(1, "sk_X509_new_null"); - for (; a->parent != NULL; a = a->parent) { + for (; a != NULL; a = a->parent) { assert(a->cert->x509 != NULL); if (!sk_X509_push(*chain, a->cert->x509)) errx(1, "sk_X509_push"); @@ -439,29 +456,122 @@ build_chain(const struct auth *a, STACK_ } /* + * Find a CRL based on the auth SKI value. + */ +static struct crl * +get_crl(const struct auth *a) +{ + struct crl find; + + if (a == NULL) + return NULL; + + find.aki = a->cert->ski; + return RB_FIND(crl_tree, &crlt, &find); +} + +/* * Add the CRL based on the certs SKI value. * No need to insert any other CRL since those were already checked. */ static void -build_crls(const struct auth *a, struct crl_tree *crlt, - STACK_OF(X509_CRL) **crls) +build_crls(const struct crl *crl, STACK_OF(X509_CRL) **crls) { - struct crl find, *found; - *crls = NULL; - if (a == NULL) + if (crl == NULL) return; if ((*crls = sk_X509_CRL_new_null()) == NULL) errx(1, "sk_X509_CRL_new_null"); - find.aki = a->cert->ski; - found = RB_FIND(crl_tree, crlt, &find); - if (found && !sk_X509_CRL_push(*crls, found->x509_crl)) + if (!sk_X509_CRL_push(*crls, crl->x509_crl)) err(1, "sk_X509_CRL_push"); } +static void +parse_entity(struct entityq *q, struct msgbuf *msgq) +{ + struct entity *entp; + struct tal *tal; + struct cert *cert; + struct mft *mft; + struct roa *roa; + struct ibuf *b; + unsigned char *f; + size_t flen; + int c; + + while ((entp = TAILQ_FIRST(q)) != NULL) { + TAILQ_REMOVE(q, entp, entries); + + b = io_new_buffer(); + io_simple_buffer(b, &entp->type, sizeof(entp->type)); + + f = NULL; + if (entp->type != RTYPE_TAL) { + f = load_file(entp->file, &flen); + if (f == NULL) + warn("%s", entp->file); + } + + switch (entp->type) { + case RTYPE_TAL: + if ((tal = tal_parse(entp->file, entp->data, + entp->datasz)) == NULL) + errx(1, "%s: could not parse tal file", + entp->file); + tal->id = entp->talid; + tal_buffer(b, tal); + tal_free(tal); + break; + case RTYPE_CER: + if (entp->has_data) + cert = proc_parser_root_cert(entp, f, flen); + else + cert = proc_parser_cert(entp, f, flen); + c = (cert != NULL); + io_simple_buffer(b, &c, sizeof(int)); + if (cert != NULL) + cert_buffer(b, cert); + /* + * The parsed certificate data "cert" is now + * managed in the "auths" table, so don't free + * it here (see the loop after "out"). + */ + break; + case RTYPE_CRL: + proc_parser_crl(entp, f, flen); + break; + case RTYPE_MFT: + mft = proc_parser_mft(entp, f, flen); + c = (mft != NULL); + io_simple_buffer(b, &c, sizeof(int)); + if (mft != NULL) + mft_buffer(b, mft); + mft_free(mft); + break; + case RTYPE_ROA: + roa = proc_parser_roa(entp, f, flen); + c = (roa != NULL); + io_simple_buffer(b, &c, sizeof(int)); + if (roa != NULL) + roa_buffer(b, roa); + roa_free(roa); + break; + case RTYPE_GBR: + proc_parser_gbr(entp, f, flen); + break; + default: + abort(); + } + + free(f); + io_close_buffer(msgq, b); + entity_free(entp); + } +} + /* * Process responsible for parsing and validating content. * All this process does is wait to be told about a file to parse, then @@ -472,29 +582,20 @@ build_crls(const struct auth *a, struct void proc_parser(int fd) { - struct tal *tal; - struct cert *cert; - struct mft *mft; - struct roa *roa; - struct entity *entp; struct entityq q; - int c, rc = 1; struct msgbuf msgq; struct pollfd pfd; - struct ibuf *b; - X509_STORE *store; - X509_STORE_CTX *ctx; - struct auth_tree auths = RB_INITIALIZER(&auths); - struct crl_tree crlt = RB_INITIALIZER(&crlt); + struct entity *entp; + struct ibuf *b, *inbuf = NULL; ERR_load_crypto_strings(); OpenSSL_add_all_ciphers(); OpenSSL_add_all_digests(); - if ((store = X509_STORE_new()) == NULL) - cryptoerrx("X509_STORE_new"); if ((ctx = X509_STORE_CTX_new()) == NULL) cryptoerrx("X509_STORE_CTX_new"); + if ((store = X509_STORE_new()) == NULL) + cryptoerrx("X509_STORE_new"); TAILQ_INIT(&q); @@ -503,8 +604,6 @@ proc_parser(int fd) pfd.fd = fd; - io_socket_nonblocking(pfd.fd); - for (;;) { pfd.events = POLLIN; if (msgq.queued) @@ -520,22 +619,16 @@ proc_parser(int fd) if ((pfd.revents & POLLHUP)) break; - /* - * Start with read events. - * This means that the parent process is sending us - * something we need to parse. - * We don't actually parse it til we have space in our - * outgoing buffer for responding, though. - */ - if ((pfd.revents & POLLIN)) { - io_socket_blocking(fd); - entp = calloc(1, sizeof(struct entity)); - if (entp == NULL) - err(1, NULL); - entity_read_req(fd, entp); - TAILQ_INSERT_TAIL(&q, entp, entries); - io_socket_nonblocking(fd); + b = io_buf_read(fd, &inbuf); + if (b != NULL) { + entp = calloc(1, sizeof(struct entity)); + if (entp == NULL) + err(1, NULL); + entity_read_req(b, entp); + TAILQ_INSERT_TAIL(&q, entp, entries); + ibuf_free(b); + } } if (pfd.revents & POLLOUT) { @@ -547,80 +640,9 @@ proc_parser(int fd) } } - /* - * If there's nothing to parse, then stop waiting for - * the write signal. - */ - - if (TAILQ_EMPTY(&q)) { - pfd.events &= ~POLLOUT; - continue; - } - - entp = TAILQ_FIRST(&q); - assert(entp != NULL); - - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); - io_simple_buffer(b, &entp->type, sizeof(entp->type)); - - switch (entp->type) { - case RTYPE_TAL: - if ((tal = tal_parse(entp->file, entp->descr)) == NULL) - goto out; - tal_buffer(b, tal); - tal_free(tal); - break; - case RTYPE_CER: - if (entp->has_pkey) - cert = proc_parser_root_cert(entp, store, ctx, - &auths, &crlt); - else - cert = proc_parser_cert(entp, store, ctx, - &auths, &crlt); - c = (cert != NULL); - io_simple_buffer(b, &c, sizeof(int)); - if (cert != NULL) - cert_buffer(b, cert); - /* - * The parsed certificate data "cert" is now - * managed in the "auths" table, so don't free - * it here (see the loop after "out"). - */ - break; - case RTYPE_MFT: - mft = proc_parser_mft(entp, store, ctx, &auths, &crlt); - c = (mft != NULL); - io_simple_buffer(b, &c, sizeof(int)); - if (mft != NULL) - mft_buffer(b, mft); - mft_free(mft); - break; - case RTYPE_CRL: - proc_parser_crl(entp, store, ctx, &crlt); - break; - case RTYPE_ROA: - roa = proc_parser_roa(entp, store, ctx, &auths, &crlt); - c = (roa != NULL); - io_simple_buffer(b, &c, sizeof(int)); - if (roa != NULL) - roa_buffer(b, roa); - roa_free(roa); - break; - case RTYPE_GBR: - proc_parser_gbr(entp, store, ctx, &auths, &crlt); - break; - default: - abort(); - } - - ibuf_close(&msgq, b); - TAILQ_REMOVE(&q, entp, entries); - entity_free(entp); + parse_entity(&q, &msgq); } - rc = 0; -out: while ((entp = TAILQ_FIRST(&q)) != NULL) { TAILQ_REMOVE(&q, entp, entries); entity_free(entp); @@ -629,9 +651,7 @@ out: /* XXX free auths and crl tree */ X509_STORE_CTX_free(ctx); - X509_STORE_free(store); - msgbuf_clear(&msgq); - exit(rc); + exit(0); } Index: usr.sbin/rpki-client/print.c =================================================================== RCS file: usr.sbin/rpki-client/print.c diff -N usr.sbin/rpki-client/print.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ usr.sbin/rpki-client/print.c 6 Nov 2021 14:21:39 -0000 @@ -0,0 +1,166 @@ +/* $OpenBSD: print.c,v 1.2 2021/10/25 14:07:56 claudio Exp $ */ +/* + * Copyright (c) 2021 Claudio Jeker + * Copyright (c) 2019 Kristaps Dzonsons + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "extern.h" + +static const char * +pretty_key_id(char *hex) +{ + static char buf[128]; /* bigger than SHA_DIGEST_LENGTH * 3 */ + size_t i; + + for (i = 0; i < sizeof(buf) && *hex != '\0'; i++) { + if (i % 3 == 2 && *hex != '\0') + buf[i] = ':'; + else + buf[i] = *hex++; + } + if (i == sizeof(buf)) + memcpy(buf + sizeof(buf) - 4, "...", 4); + return buf; +} + +void +tal_print(const struct tal *p) +{ + size_t i; + + for (i = 0; i < p->urisz; i++) + printf("%5zu: URI: %s\n", i + 1, p->uri[i]); +} + +void +cert_print(const struct cert *p) +{ + size_t i; + char buf1[64], buf2[64]; + int sockt; + char tbuf[21]; + + printf("Subject key identifier: %s\n", pretty_key_id(p->ski)); + if (p->aki != NULL) + printf("Authority key identifier: %s\n", pretty_key_id(p->aki)); + if (p->aia != NULL) + printf("Authority info access: %s\n", p->aia); + if (p->mft != NULL) + printf("Manifest: %s\n", p->mft); + if (p->repo != NULL) + printf("caRepository: %s\n", p->repo); + if (p->notify != NULL) + printf("Notify URL: %s\n", p->notify); + if (p->pubkey != NULL) + printf("BGPsec P-256 ECDSA public key: %s\n", p->pubkey); + strftime(tbuf, sizeof(tbuf), "%FT%TZ", gmtime(&p->expires)); + printf("Valid until: %s\n", tbuf); + + printf("Subordinate Resources:\n"); + + for (i = 0; i < p->asz; i++) + switch (p->as[i].type) { + case CERT_AS_ID: + printf("%5zu: AS: %u\n", i + 1, p->as[i].id); + break; + case CERT_AS_INHERIT: + printf("%5zu: AS: inherit\n", i + 1); + break; + case CERT_AS_RANGE: + printf("%5zu: AS: %u -- %u\n", i + 1, + p->as[i].range.min, p->as[i].range.max); + break; + } + + for (i = 0; i < p->ipsz; i++) + switch (p->ips[i].type) { + case CERT_IP_INHERIT: + printf("%5zu: IP: inherit\n", i + 1); + break; + case CERT_IP_ADDR: + ip_addr_print(&p->ips[i].ip, + p->ips[i].afi, buf1, sizeof(buf1)); + printf("%5zu: IP: %s\n", i + 1, buf1); + break; + case CERT_IP_RANGE: + sockt = (p->ips[i].afi == AFI_IPV4) ? + AF_INET : AF_INET6; + inet_ntop(sockt, p->ips[i].min, buf1, sizeof(buf1)); + inet_ntop(sockt, p->ips[i].max, buf2, sizeof(buf2)); + printf("%5zu: IP: %s -- %s\n", i + 1, buf1, buf2); + break; + } + +} + +void +mft_print(const struct mft *p) +{ + size_t i; + char *hash; + + printf("Subject key identifier: %s\n", pretty_key_id(p->ski)); + printf("Authority key identifier: %s\n", pretty_key_id(p->aki)); + printf("Authority info access: %s\n", p->aia); + printf("Manifest Number: %s\n", p->seqnum); + for (i = 0; i < p->filesz; i++) { + if (base64_encode(p->files[i].hash, sizeof(p->files[i].hash), + &hash) == -1) + errx(1, "base64_encode failure"); + printf("%5zu: %s\n", i + 1, p->files[i].file); + printf("\thash %s\n", hash); + free(hash); + } +} + +void +roa_print(const struct roa *p) +{ + char buf[128]; + size_t i; + char tbuf[21]; + + printf("Subject key identifier: %s\n", pretty_key_id(p->ski)); + printf("Authority key identifier: %s\n", pretty_key_id(p->aki)); + printf("Authority info access: %s\n", p->aia); + strftime(tbuf, sizeof(tbuf), "%FT%TZ", gmtime(&p->expires)); + printf("ROA valid until: %s\n", tbuf); + + printf("asID: %u\n", p->asid); + for (i = 0; i < p->ipsz; i++) { + ip_addr_print(&p->ips[i].addr, + p->ips[i].afi, buf, sizeof(buf)); + printf("%5zu: %s maxlen: %zu\n", i + 1, + buf, p->ips[i].maxlength); + } +} + +void +gbr_print(const struct gbr *p) +{ + printf("Subject key identifier: %s\n", pretty_key_id(p->ski)); + printf("Authority key identifier: %s\n", pretty_key_id(p->aki)); + printf("Authority info access: %s\n", p->aia); + printf("vcard:\n%s", p->vcard); +} Index: usr.sbin/rpki-client/repo.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/repo.c,v retrieving revision 1.5 diff -u -p -u -r1.5 repo.c --- usr.sbin/rpki-client/repo.c 13 Apr 2021 13:35:59 -0000 1.5 +++ usr.sbin/rpki-client/repo.c 6 Nov 2021 18:19:49 -0000 @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ extern struct stats stats; extern int noop; +extern int rrdpon; enum repo_state { REPO_LOADING = 0, @@ -87,11 +89,14 @@ SLIST_HEAD(, tarepo) tarepos = SLIST_HEA struct repo { SLIST_ENTRY(repo) entry; - char *repouri; /* CA repository base URI */ + char *repouri; + char *notifyuri; const struct rrdprepo *rrdp; const struct rsyncrepo *rsync; const struct tarepo *ta; struct entityq queue; /* files waiting for repo */ + time_t alarm; /* sync timeout */ + int talid; size_t id; /* identifier */ }; SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); @@ -247,7 +252,7 @@ rsync_dir(const char *uri, const char *d * Function to create all missing directories to a path. * This functions alters the path temporarily. */ -static void +static int repo_mkpath(char *file) { char *slash; @@ -256,9 +261,12 @@ repo_mkpath(char *file) slash = strrchr(file, '/'); assert(slash != NULL); *slash = '\0'; - if (mkpath(file) == -1) - err(1, "%s", file); + if (mkpath(file) == -1) { + warn("mkpath %s", file); + return -1; + } *slash = '/'; + return 0; } /* @@ -326,7 +334,25 @@ rrdp_state_filename(const struct rrdprep static void ta_fetch(struct tarepo *tr) { - int fd; + if (!rrdpon) { + for (; tr->uriidx < tr->urisz; tr->uriidx++) { + if (strncasecmp(tr->uri[tr->uriidx], + "rsync://", 8) == 0) + break; + } + } + + if (tr->uriidx >= tr->urisz) { + struct repo *rp; + + tr->state = REPO_FAILED; + logx("ta/%s: fallback to cache", tr->descr); + + SLIST_FOREACH(rp, &repos, entry) + if (rp->ta == tr) + entityq_flush(&rp->queue, rp); + return; + } logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); @@ -337,11 +363,14 @@ ta_fetch(struct tarepo *tr) */ rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir); } else { + int fd; + tr->temp = ta_filename(tr, 1); fd = mkostemp(tr->temp, O_CLOEXEC); if (fd == -1) { - err(1, "mkostemp: %s", tr->temp); - /* XXX switch to soft fail and restart with next file */ + warn("mkostemp: %s", tr->temp); + http_finish(tr->id, HTTP_FAILED, NULL); + return; } if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) warn("fchmod: %s", tr->temp); @@ -376,16 +405,17 @@ ta_get(struct tal *tal) tal->urisz = 0; tal->uri = NULL; - /* create base directory */ - if (mkpath(tr->basedir) == -1) - err(1, "%s", tr->basedir); - if (noop) { tr->state = REPO_DONE; logx("ta/%s: using cache", tr->descr); /* there is nothing in the queue so no need to flush */ - } else + } else { + /* try to create base directory */ + if (mkpath(tr->basedir) == -1) + warn("mkpath %s", tr->basedir); + ta_fetch(tr); + } return tr; } @@ -440,15 +470,18 @@ rsync_get(const char *uri) rr->repouri = repo; rr->basedir = rsync_dir(repo, "rsync"); - /* create base directory */ - if (mkpath(rr->basedir) == -1) - err(1, "%s", rr->basedir); - if (noop) { rr->state = REPO_DONE; logx("%s: using cache", rr->basedir); /* there is nothing in the queue so no need to flush */ } else { + /* create base directory */ + if (mkpath(rr->basedir) == -1) { + warn("mkpath %s", rr->basedir); + rsync_finish(rr->id, 0); + return rr; + } + logx("%s: pulling from %s", rr->basedir, rr->repouri); rsync_fetch(rr->id, rr->repouri, rr->basedir); } @@ -480,7 +513,7 @@ rsync_free(void) } } -static void rrdprepo_fetch(struct rrdprepo *); +static int rrdprepo_fetch(struct rrdprepo *); static struct rrdprepo * rrdp_get(const char *uri) @@ -507,17 +540,23 @@ rrdp_get(const char *uri) RB_INIT(&rr->added); RB_INIT(&rr->deleted); - /* create base directory */ - if (mkpath(rr->basedir) == -1) - err(1, "%s", rr->basedir); - if (noop) { rr->state = REPO_DONE; logx("%s: using cache", rr->notifyuri); /* there is nothing in the queue so no need to flush */ } else { + /* create base directory */ + if (mkpath(rr->basedir) == -1) { + warn("mkpath %s", rr->basedir); + rrdp_finish(rr->id, 0); + return rr; + } + if (rrdprepo_fetch(rr) == -1) { + rrdp_finish(rr->id, 0); + return rr; + } + logx("%s: pulling from %s", rr->notifyuri, "network"); - rrdprepo_fetch(rr); } return rr; @@ -553,30 +592,41 @@ rrdp_free(void) } } -static int +static struct rrdprepo * rrdp_basedir(const char *dir) { struct rrdprepo *rr; SLIST_FOREACH(rr, &rrdprepos, entry) - if (strcmp(dir, rr->basedir) == 0) - return 1; + if (strcmp(dir, rr->basedir) == 0) { + if (rr->state == REPO_FAILED) + return NULL; + return rr; + } - return 0; + return NULL; } /* * Allocate and insert a new repository. */ static struct repo * -repo_alloc(void) +repo_alloc(int talid) { struct repo *rp; + if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { + if (talrepocnt[talid] == MAX_REPO_PER_TAL) + warnx("too many repositories under %s", tals[talid]); + return NULL; + } + if ((rp = calloc(1, sizeof(*rp))) == NULL) err(1, NULL); rp->id = ++repoid; + rp->talid = talid; + rp->alarm = getmonotime() + MAX_REPO_TIMEOUT; TAILQ_INIT(&rp->queue); SLIST_INSERT_HEAD(&repos, rp, entry); @@ -599,23 +649,6 @@ repo_state(struct repo *rp) errx(1, "%s: bad repo", rp->repouri); } -#if 0 -/* - * locate a repository by ID. - */ -static struct repo * -repo_find(size_t id) -{ - struct repo *rp; - - SLIST_FOREACH(rp, &repos, entry) - if (id == rp->id) - break; - return rp; -} -#endif - - /* * Parse the RRDP state file if it exists and set the session struct * based on that information. @@ -697,8 +730,10 @@ rrdp_save_state(size_t id, struct rrdp_s file = rrdp_state_filename(rr, 0); temp = rrdp_state_filename(rr, 1); - if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) - err(1, "%s: mkostemp: %s", rr->basedir, temp); + if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) { + warn("mkostemp %s", temp); + goto fail; + } (void) fchmod(fd, 0644); f = fdopen(fd, "w"); if (f == NULL) @@ -733,6 +768,12 @@ fail: free(file); } +/* + * Write a file into the temporary RRDP dir but only after checking + * its hash (if required). The function also makes sure that the file + * tracking is properly adjusted. + * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error + */ int rrdp_handle_file(size_t id, enum publish_type pt, char *uri, char *hash, size_t hlen, char *data, size_t dlen) @@ -741,17 +782,13 @@ rrdp_handle_file(size_t id, enum publish struct filepath *fp; ssize_t s; char *fn; - int fd; + int fd = -1; rr = rrdp_find(id); if (rr == NULL) errx(1, "non-existant rrdp repo %zu", id); - - /* belt and suspenders */ - if (!valid_uri(uri, strlen(uri), "rsync://")) { - warnx("%s: bad file URI", rr->basedir); - return 0; - } + if (rr->state == REPO_FAILED) + return -1; if (pt == PUB_UPD || pt == PUB_DEL) { if (filepath_exists(&rr->deleted, uri)) { @@ -786,96 +823,94 @@ rrdp_handle_file(size_t id, enum publish if ((fn = rrdp_filename(rr, uri, 1)) == NULL) return 0; - repo_mkpath(fn); + if (repo_mkpath(fn) == -1) + goto fail; + fd = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0644); if (fd == -1) { warn("open %s", fn); - free(fn); - return 0; + goto fail; } if ((s = write(fd, data, dlen)) == -1) { warn("write %s", fn); - free(fn); - close(fd); - return 0; + goto fail; } close(fd); - if ((size_t)s != dlen) { - warnx("short write %s", fn); - free(fn); - return 0; - } + if ((size_t)s != dlen) /* impossible */ + errx(1, "short write %s", fn); free(fn); filepath_add(&rr->added, uri); } return 1; + +fail: + rr->state = REPO_FAILED; + if (fd != -1) + close(fd); + free(fn); + return -1; } /* * Initiate a RRDP sync, create the required temporary directory and * parse a possible state file before sending the request to the RRDP process. */ -static void +static int rrdprepo_fetch(struct rrdprepo *rr) { struct rrdp_session state = { 0 }; if (asprintf(&rr->temp, "%s.XXXXXXXX", rr->basedir) == -1) err(1, NULL); - if (mkdtemp(rr->temp) == NULL) - err(1, "mkdtemp %s", rr->temp); + if (mkdtemp(rr->temp) == NULL) { + warn("mkdtemp %s", rr->temp); + return -1; + } rrdp_parse_state(rr, &state); rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state); free(state.session_id); free(state.last_mod); + + return 0; } -static void +static int rrdp_merge_repo(struct rrdprepo *rr) { struct filepath *fp, *nfp; char *fn, *rfn; - /* XXX should delay deletes */ - RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { + RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { fn = rrdp_filename(rr, fp->file, 1); rfn = rrdp_filename(rr, fp->file, 0); if (fn == NULL || rfn == NULL) errx(1, "bad filepath"); /* should not happen */ - if (unlink(rfn) == -1) { - if (errno == ENOENT) { - if (unlink(fn) == -1) - warn("%s: unlink", fn); - } else - warn("%s: unlink", rfn); + if (repo_mkpath(rfn) == -1) { + goto fail; } - free(rfn); - free(fn); - filepath_put(&rr->deleted, fp); - } - - RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { - fn = rrdp_filename(rr, fp->file, 1); - rfn = rrdp_filename(rr, fp->file, 0); - - if (fn == NULL || rfn == NULL) - errx(1, "bad filepath"); /* should not happen */ - - repo_mkpath(rfn); - if (rename(fn, rfn) == -1) - warn("%s: rename", rfn); + if (rename(fn, rfn) == -1) { + warn("rename %s", rfn); + goto fail; + } free(rfn); free(fn); filepath_put(&rr->added, fp); } + + return 1; + +fail: + free(rfn); + free(fn); + return 0; } static void @@ -889,7 +924,7 @@ rrdp_clean_temp(struct rrdprepo *rr) RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) { if ((fn = rrdp_filename(rr, fp->file, 1)) != NULL) { if (unlink(fn) == -1) - warn("%s: unlink", fn); + warn("unlink %s", fn); free(fn); } filepath_put(&rr->added, fp); @@ -908,20 +943,19 @@ rsync_finish(size_t id, int ok) tr = ta_find(id); if (tr != NULL) { + /* repository changed state already, ignore request */ + if (tr->state != REPO_LOADING) + return; if (ok) { logx("ta/%s: loaded from network", tr->descr); stats.rsync_repos++; tr->state = REPO_DONE; - } else if (++tr->uriidx < tr->urisz) { - logx("ta/%s: load from network failed, retry", - tr->descr); - ta_fetch(tr); - return; } else { - logx("ta/%s: load from network failed, " - "fallback to cache", tr->descr); + logx("ta/%s: load from network failed", tr->descr); stats.rsync_fails++; - tr->state = REPO_FAILED; + tr->uriidx++; + ta_fetch(tr); + return; } SLIST_FOREACH(rp, &repos, entry) if (rp->ta == tr) @@ -934,6 +968,9 @@ rsync_finish(size_t id, int ok) if (rr == NULL) errx(1, "unknown rsync repo %zu", id); + /* repository changed state already, ignore request */ + if (rr->state != REPO_LOADING) + return; if (ok) { logx("%s: loaded from network", rr->basedir); stats.rsync_repos++; @@ -962,16 +999,18 @@ rrdp_finish(size_t id, int ok) rr = rrdp_find(id); if (rr == NULL) errx(1, "unknown RRDP repo %zu", id); + /* repository changed state already, ignore request */ + if (rr->state != REPO_LOADING) + return; - if (ok) { - rrdp_merge_repo(rr); + if (ok && rrdp_merge_repo(rr)) { logx("%s: loaded from network", rr->notifyuri); rr->state = REPO_DONE; stats.rrdp_repos++; SLIST_FOREACH(rp, &repos, entry) if (rp->rrdp == rr) entityq_flush(&rp->queue, rp); - } else { + } else if (!ok) { rrdp_clean_temp(rr); stats.rrdp_fails++; rr->state = REPO_FAILED; @@ -985,6 +1024,14 @@ rrdp_finish(size_t id, int ok) if (repo_state(rp) != REPO_LOADING) entityq_flush(&rp->queue, rp); } + } else { + rrdp_clean_temp(rr); + stats.rrdp_fails++; + rr->state = REPO_FAILED; + logx("%s: load from network failed", rr->notifyuri); + SLIST_FOREACH(rp, &repos, entry) + if (rp->rrdp == rr) + entityq_flush(&rp->queue, rp); } } @@ -1006,6 +1053,10 @@ http_finish(size_t id, enum http_result return; } + /* repository changed state already, ignore request */ + if (tr->state != REPO_LOADING) + return; + /* Move downloaded TA file into place, or unlink on failure. */ if (res == HTTP_OK) { char *file; @@ -1019,19 +1070,13 @@ http_finish(size_t id, enum http_result tr->state = REPO_DONE; stats.http_repos++; } else { - if (unlink(tr->temp) == -1) + if (unlink(tr->temp) == -1 && errno != ENOENT) warn("unlink %s", tr->temp); - if (++tr->uriidx < tr->urisz) { - logx("ta/%s: load from network failed, retry", - tr->descr); - ta_fetch(tr); - return; - } - - tr->state = REPO_FAILED; - logx("ta/%s: load from network failed, " - "fallback to cache", tr->descr); + tr->uriidx++; + logx("ta/%s: load from network failed", tr->descr); + ta_fetch(tr); + return; } SLIST_FOREACH(rp, &repos, entry) @@ -1045,7 +1090,7 @@ http_finish(size_t id, enum http_result * Look up a trust anchor, queueing it for download if not found. */ struct repo * -ta_lookup(struct tal *tal) +ta_lookup(int id, struct tal *tal) { struct repo *rp; @@ -1055,7 +1100,10 @@ ta_lookup(struct tal *tal) return rp; } - rp = repo_alloc(); + rp = repo_alloc(id); + if (rp == NULL) + return NULL; + if ((rp->repouri = strdup(tal->descr)) == NULL) err(1, NULL); rp->ta = ta_get(tal); @@ -1067,20 +1115,40 @@ ta_lookup(struct tal *tal) * Look up a repository, queueing it for discovery if not found. */ struct repo * -repo_lookup(const char *uri, const char *notify) +repo_lookup(int id, const char *uri, const char *notify) { - struct repo *rp; + struct repo *rp; + char *repouri; + + if ((repouri = rsync_base_uri(uri)) == NULL) + errx(1, "bad caRepository URI: %s", uri); /* Look up in repository table. */ SLIST_FOREACH(rp, &repos, entry) { - if (strcmp(rp->repouri, uri) != 0) + if (strcmp(rp->repouri, repouri) != 0) + continue; + if (rp->notifyuri != NULL) { + if (notify == NULL) + continue; + if (strcmp(rp->notifyuri, notify) != 0) + continue; + } else if (notify != NULL) continue; + /* found matching repo */ + free(repouri); return rp; } - rp = repo_alloc(); - if ((rp->repouri = strdup(uri)) == NULL) - err(1, NULL); + rp = repo_alloc(id); + if (rp == NULL) { + free(repouri); + return NULL; + } + + rp->repouri = repouri; + if (notify != NULL) + if ((rp->notifyuri = strdup(notify)) == NULL) + err(1, NULL); /* try RRDP first if available */ if (notify != NULL) @@ -1133,6 +1201,60 @@ repo_queued(struct repo *rp, struct enti return 0; } +int +repo_next_timeout(int timeout) +{ + struct repo *rp; + time_t now; + + now = getmonotime(); + /* Look up in repository table. (Lookup should actually fail here) */ + SLIST_FOREACH(rp, &repos, entry) { + if (repo_state(rp) == REPO_LOADING) { + int diff = rp->alarm - now; + diff *= 1000; + if (timeout == INFTIM || diff < timeout) + timeout = diff; + } + } + return timeout; +} + +static void +repo_fail(struct repo *rp) +{ + /* reset the alarm since code may fallback to rsync */ + rp->alarm = getmonotime() + MAX_REPO_TIMEOUT; + + if (rp->ta) + http_finish(rp->ta->id, HTTP_FAILED, NULL); + else if (rp->rrdp) + rrdp_finish(rp->rrdp->id, 0); + else if (rp->rsync) + rsync_finish(rp->rsync->id, 0); + else + errx(1, "%s: bad repo", rp->repouri); +} + +void +repo_check_timeout(void) +{ + struct repo *rp; + time_t now; + + now = getmonotime(); + /* Look up in repository table. (Lookup should actually fail here) */ + SLIST_FOREACH(rp, &repos, entry) { + if (repo_state(rp) == REPO_LOADING) { + if (rp->alarm <= now) { + warnx("%s: synchronisation timeout", + rp->repouri); + repo_fail(rp); + } + } + } +} + static char ** add_to_del(char **del, size_t *dsz, char *file) { @@ -1146,19 +1268,43 @@ add_to_del(char **del, size_t *dsz, char *dsz = i + 1; return del; } + +static char ** +repo_rrdp_cleanup(struct filepath_tree *tree, struct rrdprepo *rr, + char **del, size_t *delsz) +{ + struct filepath *fp, *nfp; + char *fn; + + RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { + fn = rrdp_filename(rr, fp->file, 0); + /* temp dir will be cleaned up by repo_cleanup() */ + + if (fn == NULL) + errx(1, "bad filepath"); /* should not happen */ + + if (!filepath_exists(tree, fn)) + del = add_to_del(del, delsz, fn); + else + warnx("%s: referenced file supposed to be deleted", fn); + + free(fn); + filepath_put(&rr->deleted, fp); + } + + return del; +} + void repo_cleanup(struct filepath_tree *tree) { - size_t i, delsz = 0, dirsz = 0; + size_t i, cnt, delsz = 0, dirsz = 0; char **del = NULL, **dir = NULL; - char *argv[4]; + char *argv[4] = { "ta", "rsync", "rrdp", NULL }; + struct rrdprepo *rr; FTS *fts; FTSENT *e; - argv[0] = "ta"; - argv[1] = "rsync"; - argv[2] = "rrdp"; - argv[3] = NULL; if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) err(1, "fts_open"); errno = 0; @@ -1170,10 +1316,12 @@ repo_cleanup(struct filepath_tree *tree) e->fts_path); break; case FTS_D: - /* skip rrdp base directories during cleanup */ - if (rrdp_basedir(e->fts_path)) + /* special cleanup for rrdp directories */ + if ((rr = rrdp_basedir(e->fts_path)) != NULL) { + del = repo_rrdp_cleanup(tree, rr, del, &delsz); if (fts_set(fts, e, FTS_SKIP) == -1) err(1, "fts_set"); + } break; case FTS_DP: if (!filepath_dir_exists(tree, e->fts_path)) @@ -1207,25 +1355,31 @@ repo_cleanup(struct filepath_tree *tree) if (fts_close(fts) == -1) err(1, "fts_close"); + cnt = 0; for (i = 0; i < delsz; i++) { - if (unlink(del[i]) == -1) - warn("unlink %s", del[i]); - if (verbose > 1) - logx("deleted %s", del[i]); + if (unlink(del[i]) == -1) { + if (errno != ENOENT) + warn("unlink %s", del[i]); + } else { + if (verbose > 1) + logx("deleted %s", del[i]); + cnt++; + } free(del[i]); } free(del); - stats.del_files = delsz; + stats.del_files = cnt; + cnt = 0; for (i = 0; i < dirsz; i++) { if (rmdir(dir[i]) == -1) warn("rmdir %s", dir[i]); - if (verbose > 1) - logx("deleted dir %s", dir[i]); + else + cnt++; free(dir[i]); } free(dir); - stats.del_dirs = dirsz; + stats.del_dirs = cnt; } void Index: usr.sbin/rpki-client/roa.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/roa.c,v retrieving revision 1.17 diff -u -p -u -r1.17 roa.c --- usr.sbin/rpki-client/roa.c 29 Mar 2021 06:50:44 -0000 1.17 +++ usr.sbin/rpki-client/roa.c 6 Nov 2021 18:20:05 -0000 @@ -36,6 +36,8 @@ struct parse { struct roa *res; /* results */ }; +static ASN1_OBJECT *roa_oid; + /* * Parse IP address (ROAIPAddress), RFC 6482, section 3.3. * Returns zero on failure, non-zero on success. @@ -49,6 +51,7 @@ roa_parse_addr(const ASN1_OCTET_STRING * int rc = 0; const ASN1_TYPE *t; const ASN1_INTEGER *maxlength = NULL; + long maxlen; struct ip_addr addr; struct roa_ip *res; @@ -81,11 +84,6 @@ roa_parse_addr(const ASN1_OCTET_STRING * goto out; } - /* - * RFC 6482, section 3.3 doesn't ever actually state that the - * maximum length can't be negative, but it needs to be >=0. - */ - if (sk_ASN1_TYPE_num(seq) == 2) { t = sk_ASN1_TYPE_value(seq, 1); if (t->type != V_ASN1_INTEGER) { @@ -94,33 +92,30 @@ roa_parse_addr(const ASN1_OCTET_STRING * p->fn, ASN1_tag2str(t->type), t->type); goto out; } - maxlength = t->value.integer; - /* - * It's safe to use ASN1_INTEGER_get() here - * because we're not going to have more than signed 32 - * bit maximum of length. - */ - - if (ASN1_INTEGER_get(maxlength) < 0) { + maxlength = t->value.integer; + maxlen = ASN1_INTEGER_get(maxlength); + if (maxlen < 0) { warnx("%s: RFC 6482 section 3.2: maxLength: " - "want positive integer, have %ld", - p->fn, ASN1_INTEGER_get(maxlength)); + "want positive integer, have %ld", p->fn, maxlen); + goto out; + } + if (addr.prefixlen > maxlen) { + warnx("%s: prefixlen (%d) larger than maxLength (%ld)", + p->fn, addr.prefixlen, maxlen); + goto out; + } + if (maxlen > ((afi == AFI_IPV4) ? 32 : 128)) { + warnx("%s: maxLength (%ld) too large", p->fn, maxlen); goto out; } - /* FIXME: maximum check. */ } - p->res->ips = recallocarray(p->res->ips, p->res->ipsz, p->res->ipsz + 1, - sizeof(struct roa_ip)); - if (p->res->ips == NULL) - err(1, NULL); res = &p->res->ips[p->res->ipsz++]; res->addr = addr; res->afi = afi; - res->maxlength = (maxlength == NULL) ? addr.prefixlen : - ASN1_INTEGER_get(maxlength); + res->maxlength = (maxlength == NULL) ? addr.prefixlen : maxlen; ip_roa_compose_ranges(res); rc = 1; @@ -184,6 +179,12 @@ roa_parse_ipfam(const ASN1_OCTET_STRING goto out; } + /* will be called multiple times so use recallocarray */ + p->res->ips = recallocarray(p->res->ips, p->res->ipsz, + p->res->ipsz + sk_ASN1_TYPE_num(sseq), sizeof(struct roa_ip)); + if (p->res->ips == NULL) + err(1, NULL); + for (i = 0; i < sk_ASN1_TYPE_num(sseq); i++) { t = sk_ASN1_TYPE_value(sseq, i); if (t->type != V_ASN1_SEQUENCE) { @@ -249,6 +250,7 @@ roa_parse_econtent(const unsigned char * ASN1_SEQUENCE_ANY *seq; int i = 0, rc = 0, sz; const ASN1_TYPE *t; + long roa_version; /* RFC 6482, section 3. */ @@ -265,26 +267,22 @@ roa_parse_econtent(const unsigned char * goto out; } - /* RFC 6482, section 3.1. */ - + /* Parse the optional version field */ if (sz == 3) { t = sk_ASN1_TYPE_value(seq, i++); + d = t->value.asn1_string->data; + dsz = t->value.asn1_string->length; - /* - * This check with ASN1_INTEGER_get() is fine since - * we're looking for a value of zero anyway, so any - * overflowing number will be definition be wrong. - */ + if (cms_econtent_version(p->fn, &d, dsz, &roa_version) == -1) + goto out; - if (t->type != V_ASN1_INTEGER) { - warnx("%s: RFC 6482 section 3.1: version: " - "want ASN.1 integer, have %s (NID %d)", - p->fn, ASN1_tag2str(t->type), t->type); + switch (roa_version) { + case 0: + warnx("%s: incorrect encoding for version 0", p->fn); goto out; - } else if (ASN1_INTEGER_get(t->value.integer) != 0) { - warnx("%s: RFC 6482 section 3.1: version: " - "want version 0, have %ld", - p->fn, ASN1_INTEGER_get(t->value.integer)); + default: + warnx("%s: version %ld not supported (yet)", p->fn, + roa_version); goto out; } } @@ -329,20 +327,28 @@ out: * Returns the ROA or NULL if the document was malformed. */ struct roa * -roa_parse(X509 **x509, const char *fn) +roa_parse(X509 **x509, const char *fn, const unsigned char *der, size_t len) { struct parse p; size_t cmsz; unsigned char *cms; int rc = 0; + const ASN1_TIME *at; + struct tm expires_tm; + time_t expires; memset(&p, 0, sizeof(struct parse)); p.fn = fn; /* OID from section 2, RFC 6482. */ + if (roa_oid == NULL) { + roa_oid = OBJ_txt2obj("1.2.840.113549.1.9.16.1.24", 1); + if (roa_oid == NULL) + errx(1, "OBJ_txt2obj for %s failed", + "1.2.840.113549.1.9.16.1.24"); + } - cms = cms_parse_validate(x509, fn, - "1.2.840.113549.1.9.16.1.24", &cmsz); + cms = cms_parse_validate(x509, fn, der, len, roa_oid, &cmsz); if (cms == NULL) return NULL; @@ -358,6 +364,21 @@ roa_parse(X509 **x509, const char *fn) goto out; } + at = X509_get0_notAfter(*x509); + if (at == NULL) { + warnx("%s: X509_get0_notAfter failed", fn); + goto out; + } + memset(&expires_tm, 0, sizeof(expires_tm)); + if (ASN1_time_parse(at->data, at->length, &expires_tm, 0) == -1) { + warnx("%s: ASN1_time_parse failed", fn); + goto out; + } + if ((expires = mktime(&expires_tm)) == -1) + errx(1, "mktime failed"); + + p.res->expires = expires; + if (!roa_parse_econtent(cms, cmsz, &p)) goto out; @@ -388,7 +409,6 @@ roa_free(struct roa *p) free(p->aki); free(p->ski); free(p->ips); - free(p->tal); free(p); } @@ -399,24 +419,17 @@ roa_free(struct roa *p) void roa_buffer(struct ibuf *b, const struct roa *p) { - size_t i; + io_simple_buffer(b, &p->valid, sizeof(p->valid)); + io_simple_buffer(b, &p->asid, sizeof(p->asid)); + io_simple_buffer(b, &p->talid, sizeof(p->talid)); + io_simple_buffer(b, &p->ipsz, sizeof(p->ipsz)); + io_simple_buffer(b, &p->expires, sizeof(p->expires)); - io_simple_buffer(b, &p->valid, sizeof(int)); - io_simple_buffer(b, &p->asid, sizeof(uint32_t)); - io_simple_buffer(b, &p->ipsz, sizeof(size_t)); - - for (i = 0; i < p->ipsz; i++) { - io_simple_buffer(b, &p->ips[i].afi, sizeof(enum afi)); - io_simple_buffer(b, &p->ips[i].maxlength, sizeof(size_t)); - io_simple_buffer(b, p->ips[i].min, sizeof(p->ips[i].min)); - io_simple_buffer(b, p->ips[i].max, sizeof(p->ips[i].max)); - ip_addr_buffer(b, &p->ips[i].addr); - } + io_simple_buffer(b, p->ips, p->ipsz * sizeof(p->ips[0])); io_str_buffer(b, p->aia); io_str_buffer(b, p->aki); io_str_buffer(b, p->ski); - io_str_buffer(b, p->tal); } /* @@ -425,34 +438,27 @@ roa_buffer(struct ibuf *b, const struct * Result must be passed to roa_free(). */ struct roa * -roa_read(int fd) +roa_read(struct ibuf *b) { struct roa *p; - size_t i; if ((p = calloc(1, sizeof(struct roa))) == NULL) err(1, NULL); - io_simple_read(fd, &p->valid, sizeof(int)); - io_simple_read(fd, &p->asid, sizeof(uint32_t)); - io_simple_read(fd, &p->ipsz, sizeof(size_t)); + io_read_buf(b, &p->valid, sizeof(p->valid)); + io_read_buf(b, &p->asid, sizeof(p->asid)); + io_read_buf(b, &p->talid, sizeof(p->talid)); + io_read_buf(b, &p->ipsz, sizeof(p->ipsz)); + io_read_buf(b, &p->expires, sizeof(p->expires)); if ((p->ips = calloc(p->ipsz, sizeof(struct roa_ip))) == NULL) err(1, NULL); + io_read_buf(b, p->ips, p->ipsz * sizeof(p->ips[0])); - for (i = 0; i < p->ipsz; i++) { - io_simple_read(fd, &p->ips[i].afi, sizeof(enum afi)); - io_simple_read(fd, &p->ips[i].maxlength, sizeof(size_t)); - io_simple_read(fd, &p->ips[i].min, sizeof(p->ips[i].min)); - io_simple_read(fd, &p->ips[i].max, sizeof(p->ips[i].max)); - ip_addr_read(fd, &p->ips[i].addr); - } - - io_str_read(fd, &p->aia); - io_str_read(fd, &p->aki); - io_str_read(fd, &p->ski); - io_str_read(fd, &p->tal); - assert(p->aia && p->aki && p->ski && p->tal); + io_read_str(b, &p->aia); + io_read_str(b, &p->aki); + io_read_str(b, &p->ski); + assert(p->aia && p->aki && p->ski); return p; } @@ -466,8 +472,8 @@ void roa_insert_vrps(struct vrp_tree *tree, struct roa *roa, size_t *vrps, size_t *uniqs) { - struct vrp *v; - size_t i; + struct vrp *v, *found; + size_t i; for (i = 0; i < roa->ipsz; i++) { if ((v = malloc(sizeof(*v))) == NULL) @@ -476,12 +482,25 @@ roa_insert_vrps(struct vrp_tree *tree, s v->addr = roa->ips[i].addr; v->maxlength = roa->ips[i].maxlength; v->asid = roa->asid; - if ((v->tal = strdup(roa->tal)) == NULL) - err(1, NULL); - if (RB_INSERT(vrp_tree, tree, v) == NULL) - (*uniqs)++; - else /* already exists */ + v->talid = roa->talid; + v->expires = roa->expires; + + /* + * Check if a similar VRP already exists in the tree. + * If the found VRP expires sooner, update it to this + * ROAs later expiry moment. + */ + if ((found = RB_INSERT(vrp_tree, tree, v)) != NULL) { + /* already exists */ + if (found->expires < v->expires) { + /* update found with preferred data */ + found->talid = v->talid; + found->expires = v->expires; + } free(v); + } else + (*uniqs)++; + (*vrps)++; } } Index: usr.sbin/rpki-client/rpki-client.8 =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rpki-client.8,v retrieving revision 1.43 diff -u -p -u -r1.43 rpki-client.8 --- usr.sbin/rpki-client/rpki-client.8 8 Apr 2021 14:03:32 -0000 1.43 +++ usr.sbin/rpki-client/rpki-client.8 6 Nov 2021 18:22:59 -0000 @@ -14,7 +14,7 @@ .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. .\" -.Dd $Mdocdate: April 8 2021 $ +.Dd $Mdocdate: October 26 2021 $ .Dt RPKI-CLIENT 8 .Os .Sh NAME @@ -66,9 +66,13 @@ with multiple interfaces. .It Fl c Create output in the file .Pa csv -in the output directory as comma-separated values of the prefix in slash notation, -the maximum prefix length, the autonomous system number, and an abbreviation -for the trust anchor the entry is derived from. +in the output directory as comma-separated values of the +.Em Autonomous System , +the prefix in slash notation, the maximum prefix length, an abbreviation for +the +.Em Trust Anchor +the entry is derived from, and the moment the VRP will expire derived from +the chain of X.509 certificates and CRLs in seconds since the Epoch, UTC. .It Fl d Ar cachedir The directory where .Nm @@ -90,11 +94,15 @@ flags and connect with rsync-protocol lo Create output in the file .Pa json in the output directory as JSON object. -This format is similar to that produced by other RPKI validators. +See +.Fl c +for a description of the fields. .It Fl n Offline mode. Validate the contents of .Ar cachedir +and write to +.Ar outputdir without synchronizing via RRDP or RSYNC. .It Fl o Create output in the file @@ -109,12 +117,11 @@ and .Fl j options are not specified this is the default. .It Fl R -Do not synchronize via RRDP. -This is the default. +Synchronize via RSYNC only. .It Fl r -Attempt to synchronize via RRDP. +Synchronize via RRDP. If RRDP fails, RSYNC will be used. -This flag is for testing purposes and will be removed in a future release. +This is the default. Mutually exclusive with .Fl n . .It Fl s Ar timeout @@ -166,8 +173,13 @@ should be run hourly by use .Xr crontab 1 to uncomment the entry in root's crontab. -.\" .Sh ENVIRONMENT -.\" For sections 1, 6, 7, and 8 only. +.Sh ENVIRONMENT +.Nm +utilizes the following environment variables: +.Bl -tag -width "http_proxy" +.It Ev http_proxy +URL of HTTP proxy to use. +.El .Sh FILES .Bl -tag -width "/var/db/rpki-client/openbgpd" -compact .It Pa /etc/rpki/*.tal @@ -181,10 +193,6 @@ default roa-set output file. .El .Sh EXIT STATUS .Ex -std -.\" For sections 1, 6, and 8 only. -.\" .Sh EXAMPLES -.\" .Sh DIAGNOSTICS -.\" For sections 1, 4, 6, 7, 8, and 9 printf/stderr messages only. .Sh SEE ALSO .Xr openrsync 1 , .Xr bgpd.conf 5 @@ -225,10 +233,13 @@ A Profile for X.509 PKIX Resource Certif Signed Object Template for the Resource Public Key Infrastructure (RPKI). .It RFC 6493 The Resource Public Key Infrastructure (RPKI) Ghostbusters Record. -.It RFC 7730 -Resource Public Key Infrastructure (RPKI) Trust Anchor Locator. .It RFC 8182 The RPKI Repository Delta Protocol (RRDP). +.It RFC 8209 +A Profile for BGPsec Router Certificates, Certificate Revocation Lists, and +Certification Requests. +.It RFC 8630 +Resource Public Key Infrastructure (RPKI) Trust Anchor Locator. .El .\" .Sh HISTORY .Sh AUTHORS Index: usr.sbin/rpki-client/rrdp.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rrdp.c,v retrieving revision 1.5 diff -u -p -u -r1.5 rrdp.c --- usr.sbin/rpki-client/rrdp.c 15 Apr 2021 13:31:30 -0000 1.5 +++ usr.sbin/rpki-client/rrdp.c 6 Nov 2021 18:21:41 -0000 @@ -80,7 +80,7 @@ struct publish_xml { char *uri; char *data; char hash[SHA256_DIGEST_LENGTH]; - int data_length; + size_t data_length; enum publish_type type; }; @@ -140,12 +140,11 @@ rrdp_done(size_t id, int ok) enum rrdp_msg type = RRDP_END; struct ibuf *b; - if ((b = ibuf_open(sizeof(type) + sizeof(id) + sizeof(ok))) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &id, sizeof(id)); io_simple_buffer(b, &ok, sizeof(ok)); - ibuf_close(&msgq, b); + io_close_buffer(&msgq, b); } /* @@ -162,13 +161,12 @@ rrdp_http_req(size_t id, const char *uri enum rrdp_msg type = RRDP_HTTP_REQ; struct ibuf *b; - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &id, sizeof(id)); io_str_buffer(b, uri); io_str_buffer(b, last_mod); - ibuf_close(&msgq, b); + io_close_buffer(&msgq, b); } /* @@ -180,14 +178,13 @@ rrdp_state_send(struct rrdp *s) enum rrdp_msg type = RRDP_SESSION; struct ibuf *b; - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &type, sizeof(type)); io_simple_buffer(b, &s->id, sizeof(s->id)); io_str_buffer(b, s->current.session_id); io_simple_buffer(b, &s->current.serial, sizeof(s->current.serial)); io_str_buffer(b, s->current.last_mod); - ibuf_close(&msgq, b); + io_close_buffer(&msgq, b); } static struct rrdp * @@ -211,7 +208,8 @@ rrdp_new(size_t id, char *local, char *n if ((s->parser = XML_ParserCreate("US-ASCII")) == NULL) err(1, "XML_ParserCreate"); - s->nxml = new_notification_xml(s->parser, &s->repository, &s->current); + s->nxml = new_notification_xml(s->parser, &s->repository, &s->current, + notify); TAILQ_INSERT_TAIL(&states, s, entry); @@ -272,6 +270,7 @@ rrdp_failed(struct rrdp *s) s->sxml = new_snapshot_xml(s->parser, &s->current, s); s->task = SNAPSHOT; s->state = RRDP_STATE_REQ; + logx("%s: delta sync failed, fallback to snapshot", s->local); } else { /* * TODO: update state to track recurring failures @@ -297,7 +296,6 @@ rrdp_finished(struct rrdp *s) return; if (s->state & RRDP_STATE_PARSE_ERROR) { - warnx("%s: failed after XML parse error", s->local); rrdp_failed(s); return; } @@ -331,22 +329,19 @@ rrdp_finished(struct rrdp *s) s->last_mod = NULL; switch (s->task) { case NOTIFICATION: - warnx("%s: repository not modified", - s->local); + logx("%s: repository not modified", s->local); rrdp_state_send(s); rrdp_free(s); rrdp_done(id, 1); break; case SNAPSHOT: - warnx("%s: downloading snapshot", - s->local); + logx("%s: downloading snapshot", s->local); s->sxml = new_snapshot_xml(p, &s->current, s); s->state = RRDP_STATE_REQ; break; case DELTA: - warnx("%s: downloading %lld deltas", - s->local, s->repository.serial - - s->current.serial); + logx("%s: downloading %lld deltas", s->local, + s->repository.serial - s->current.serial); s->dxml = new_delta_xml(p, &s->current, s); s->state = RRDP_STATE_REQ; break; @@ -372,12 +367,11 @@ rrdp_finished(struct rrdp *s) break; } } else if (s->res == HTTP_NOT_MOD && s->task == NOTIFICATION) { - warnx("%s: notification file not modified", s->local); + logx("%s: notification file not modified", s->local); /* no need to update state file */ rrdp_free(s); rrdp_done(id, 1); } else { - warnx("%s: HTTP request failed", s->local); rrdp_failed(s); } } @@ -385,31 +379,37 @@ rrdp_finished(struct rrdp *s) static void rrdp_input_handler(int fd) { + static struct ibuf *inbuf; char *local, *notify, *session_id, *last_mod; + struct ibuf *b; struct rrdp *s; enum rrdp_msg type; enum http_result res; long long serial; size_t id; - int infd, ok; + int ok; - infd = io_recvfd(fd, &type, sizeof(type)); - io_simple_read(fd, &id, sizeof(id)); + b = io_buf_recvfd(fd, &inbuf); + if (b == NULL) + return; + + io_read_buf(b, &type, sizeof(type)); + io_read_buf(b, &id, sizeof(id)); switch (type) { case RRDP_START: - io_str_read(fd, &local); - io_str_read(fd, ¬ify); - io_str_read(fd, &session_id); - io_simple_read(fd, &serial, sizeof(serial)); - io_str_read(fd, &last_mod); - if (infd != -1) - errx(1, "received unexpected fd %d", infd); + io_read_str(b, &local); + io_read_str(b, ¬ify); + io_read_str(b, &session_id); + io_read_buf(b, &serial, sizeof(serial)); + io_read_str(b, &last_mod); + if (b->fd != -1) + errx(1, "received unexpected fd"); s = rrdp_new(id, local, notify, session_id, serial, last_mod); break; case RRDP_HTTP_INI: - if (infd == -1) + if (b->fd == -1) errx(1, "expected fd not received"); s = rrdp_get(id); if (s == NULL) @@ -417,13 +417,13 @@ rrdp_input_handler(int fd) if (s->state != RRDP_STATE_WAIT) errx(1, "%s: bad internal state", s->local); - s->infd = infd; + s->infd = b->fd; s->state = RRDP_STATE_PARSE; break; case RRDP_HTTP_FIN: - io_simple_read(fd, &res, sizeof(res)); - io_str_read(fd, &last_mod); - if (infd != -1) + io_read_buf(b, &res, sizeof(res)); + io_read_str(b, &last_mod); + if (b->fd != -1) errx(1, "received unexpected fd"); s = rrdp_get(id); @@ -441,10 +441,10 @@ rrdp_input_handler(int fd) s = rrdp_get(id); if (s == NULL) errx(1, "rrdp session %zu does not exist", id); - if (infd != -1) - errx(1, "received unexpected fd %d", infd); - io_simple_read(fd, &ok, sizeof(ok)); - if (ok == 0) + if (b->fd != -1) + errx(1, "received unexpected fd"); + io_read_buf(b, &ok, sizeof(ok)); + if (ok != 1) s->file_failed++; s->file_pending--; if (s->file_pending == 0) @@ -453,6 +453,7 @@ rrdp_input_handler(int fd) default: errx(1, "unexpected message %d", type); } + ibuf_free(b); } static void @@ -512,13 +513,12 @@ proc_rrdp(int fd) if (pledge("stdio recvfd", NULL) == -1) err(1, "pledge"); - memset(&pfds, 0, sizeof(pfds)); - msgbuf_init(&msgq); msgq.fd = fd; for (;;) { i = 1; + memset(&pfds, 0, sizeof(pfds)); TAILQ_FOREACH(s, &states, entry) { if (i >= MAX_SESSIONS + 1) { /* not enough sessions, wait for better times */ @@ -564,14 +564,12 @@ proc_rrdp(int fd) if (pfds[0].revents & POLLHUP) break; if (pfds[0].revents & POLLOUT) { - io_socket_nonblocking(fd); switch (msgbuf_write(&msgq)) { case 0: errx(1, "write: connection closed"); case -1: err(1, "write"); } - io_socket_blocking(fd); } if (pfds[0].revents & POLLIN) rrdp_input_handler(fd); @@ -625,27 +623,34 @@ free_publish_xml(struct publish_xml *pxm * Add buf to the base64 data string, ensure that this remains a proper * string by NUL-terminating the string. */ -void +int publish_add_content(struct publish_xml *pxml, const char *buf, int length) { - int new_length; + size_t newlen, outlen; /* * optmisiation, this often gets called with '\n' as the * only data... seems wasteful */ if (length == 1 && buf[0] == '\n') - return; + return 0; /* append content to data */ - new_length = pxml->data_length + length; - pxml->data = realloc(pxml->data, new_length + 1); + if (SIZE_MAX - length - 1 <= pxml->data_length) + return -1; + newlen = pxml->data_length + length; + if (base64_decode_len(newlen, &outlen) == -1 || + outlen > MAX_FILE_SIZE) + return -1; + + pxml->data = realloc(pxml->data, newlen + 1); if (pxml->data == NULL) err(1, "%s", __func__); memcpy(pxml->data + pxml->data_length, buf, length); - pxml->data[new_length] = '\0'; - pxml->data_length = new_length; + pxml->data[newlen] = '\0'; + pxml->data_length = newlen; + return 0; } /* @@ -664,20 +669,23 @@ publish_done(struct rrdp *s, struct publ size_t datasz = 0; if (pxml->data_length > 0) - if ((base64_decode(pxml->data, &data, &datasz)) == -1) + if ((base64_decode(pxml->data, pxml->data_length, + &data, &datasz)) == -1) return -1; - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); - io_simple_buffer(b, &type, sizeof(type)); - io_simple_buffer(b, &s->id, sizeof(s->id)); - io_simple_buffer(b, &pxml->type, sizeof(pxml->type)); - if (pxml->type != PUB_ADD) - io_simple_buffer(b, &pxml->hash, sizeof(pxml->hash)); - io_str_buffer(b, pxml->uri); - io_buf_buffer(b, data, datasz); - ibuf_close(&msgq, b); - s->file_pending++; + /* only send files if the fetch did not fail already */ + if (s->file_failed == 0) { + b = io_new_buffer(); + io_simple_buffer(b, &type, sizeof(type)); + io_simple_buffer(b, &s->id, sizeof(s->id)); + io_simple_buffer(b, &pxml->type, sizeof(pxml->type)); + if (pxml->type != PUB_ADD) + io_simple_buffer(b, &pxml->hash, sizeof(pxml->hash)); + io_str_buffer(b, pxml->uri); + io_buf_buffer(b, data, datasz); + io_close_buffer(&msgq, b); + s->file_pending++; + } free(data); free_publish_xml(pxml); Index: usr.sbin/rpki-client/rrdp.h =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rrdp.h,v retrieving revision 1.1 diff -u -p -u -r1.1 rrdp.h --- usr.sbin/rpki-client/rrdp.h 1 Apr 2021 16:04:48 -0000 1.1 +++ usr.sbin/rpki-client/rrdp.h 6 Nov 2021 14:21:39 -0000 @@ -1,3 +1,20 @@ +/* $OpenBSD: rrdp.h,v 1.6 2021/10/29 09:27:36 claudio Exp $ */ +/* + * Copyright (c) 2020 Nils Fisher + * Copyright (c) 2021 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ #ifndef _RRDPH_ #define _RRDPH_ @@ -10,7 +27,7 @@ XML_StopParser(p, XML_FALSE); \ warnx(__VA_ARGS__); \ return; \ -} while(0) +} while (0) enum rrdp_task { NOTIFICATION, @@ -19,7 +36,7 @@ enum rrdp_task { }; /* rrdp generic */ -char *xstrdup(const char *); +char *xstrdup(const char *); int hex_decode(const char *, char *, size_t); /* publish or withdraw element */ @@ -29,7 +46,7 @@ struct publish_xml; struct publish_xml *new_publish_xml(enum publish_type, char *, char *, size_t); void free_publish_xml(struct publish_xml *); -void publish_add_content(struct publish_xml *, +int publish_add_content(struct publish_xml *, const char *, int); int publish_done(struct rrdp *, struct publish_xml *); @@ -37,8 +54,9 @@ int publish_done(struct rrdp *, struc struct notification_xml; struct notification_xml *new_notification_xml(XML_Parser, - struct rrdp_session *, struct rrdp_session *); -void free_notification_xml(struct notification_xml *); + struct rrdp_session *, struct rrdp_session *, + const char *); +void free_notification_xml(struct notification_xml *); enum rrdp_task notification_done(struct notification_xml *, char *); const char *notification_get_next(struct notification_xml *, Index: usr.sbin/rpki-client/rrdp_delta.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rrdp_delta.c,v retrieving revision 1.1 diff -u -p -u -r1.1 rrdp_delta.c --- usr.sbin/rpki-client/rrdp_delta.c 1 Apr 2021 16:04:48 -0000 1.1 +++ usr.sbin/rpki-client/rrdp_delta.c 6 Nov 2021 18:40:17 -0000 @@ -86,7 +86,7 @@ start_delta_elem(struct delta_xml *dxml, continue; } PARSE_FAIL(p, "parse failed - non conforming " - "attribute found in delta elem"); + "attribute '%s' found in delta elem", attr[i]); } if (!(has_xmlns && dxml->version && dxml->session_id && dxml->serial)) PARSE_FAIL(p, "parse failed - incomplete delta attributes"); @@ -115,7 +115,7 @@ start_publish_withdraw_elem(struct delta int withdraw) { XML_Parser p = dxml->parser; - char *uri, hash[SHA256_DIGEST_LENGTH]; + char *uri = NULL, hash[SHA256_DIGEST_LENGTH]; int i, hasUri = 0, hasHash = 0; enum publish_type pub = PUB_UPD; @@ -135,7 +135,7 @@ start_publish_withdraw_elem(struct delta continue; } PARSE_FAIL(p, "parse failed - non conforming " - "attribute found in publish/withdraw elem"); + "attribute '%s' found in publish/withdraw elem", attr[i]); } if (hasUri != 1) PARSE_FAIL(p, @@ -217,9 +217,21 @@ static void delta_content_handler(void *data, const char *content, int length) { struct delta_xml *dxml = data; + XML_Parser p = dxml->parser; if (dxml->scope == DELTA_SCOPE_PUBLISH) - publish_add_content(dxml->pxml, content, length); + if (publish_add_content(dxml->pxml, content, length) == -1) + PARSE_FAIL(p, "parse failed - content too big"); +} + +static void +delta_doctype_handler(void *data, const char *doctypeName, + const char *sysid, const char *pubid, int subset) +{ + struct delta_xml *dxml = data; + XML_Parser p = dxml->parser; + + PARSE_FAIL(p, "parse failed - DOCTYPE not allowed"); } struct delta_xml * @@ -240,6 +252,7 @@ new_delta_xml(XML_Parser p, struct rrdp_ delta_xml_elem_end); XML_SetCharacterDataHandler(dxml->parser, delta_content_handler); XML_SetUserData(dxml->parser, dxml); + XML_SetDoctypeDeclHandler(dxml->parser, delta_doctype_handler, NULL); return dxml; } Index: usr.sbin/rpki-client/rrdp_notification.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rrdp_notification.c,v retrieving revision 1.4 diff -u -p -u -r1.4 rrdp_notification.c --- usr.sbin/rpki-client/rrdp_notification.c 15 Apr 2021 08:58:46 -0000 1.4 +++ usr.sbin/rpki-client/rrdp_notification.c 6 Nov 2021 18:38:39 -0000 @@ -53,6 +53,7 @@ struct notification_xml { XML_Parser parser; struct rrdp_session *repository; struct rrdp_session *current; + const char *notifyuri; char *session_id; char *snapshot_uri; char snapshot_hash[SHA256_DIGEST_LENGTH]; @@ -139,7 +140,7 @@ start_notification_elem(struct notificat continue; } PARSE_FAIL(p, "parse failed - non conforming " - "attribute found in notification elem"); + "attribute '%s' found in notification elem", attr[i]); } if (!(has_xmlns && nxml->version && nxml->session_id && nxml->serial)) PARSE_FAIL(p, "parse failed - incomplete " @@ -171,7 +172,8 @@ start_snapshot_elem(struct notification_ for (i = 0; attr[i]; i += 2) { if (strcmp("uri", attr[i]) == 0 && hasUri++ == 0) { if (valid_uri(attr[i + 1], strlen(attr[i + 1]), - "https://")) { + "https://") && + valid_origin(attr[i + 1], nxml->notifyuri)) { nxml->snapshot_uri = xstrdup(attr[i + 1]); continue; } @@ -182,7 +184,7 @@ start_snapshot_elem(struct notification_ continue; } PARSE_FAIL(p, "parse failed - non conforming " - "attribute found in snapshot elem"); + "attribute '%s' found in snapshot elem", attr[i]); } if (hasUri != 1 || hasHash != 1) PARSE_FAIL(p, "parse failed - incomplete snapshot attributes"); @@ -216,7 +218,8 @@ start_delta_elem(struct notification_xml for (i = 0; attr[i]; i += 2) { if (strcmp("uri", attr[i]) == 0 && hasUri++ == 0) { if (valid_uri(attr[i + 1], strlen(attr[i + 1]), - "https://")) { + "https://") && + valid_origin(attr[i + 1], nxml->notifyuri)) { delta_uri = attr[i + 1]; continue; } @@ -235,7 +238,7 @@ start_delta_elem(struct notification_xml continue; } PARSE_FAIL(p, "parse failed - non conforming " - "attribute found in snapshot elem"); + "attribute '%s' found in snapshot elem", attr[i]); } /* Only add to the list if we are relevant */ if (hasUri != 1 || hasHash != 1 || delta_serial == 0) @@ -304,9 +307,19 @@ notification_xml_elem_end(void *data, co PARSE_FAIL(p, "parse failed - unexpected elem exit found"); } +static void +notification_doctype_handler(void *data, const char *doctypeName, + const char *sysid, const char *pubid, int subset) +{ + struct notification_xml *nxml = data; + XML_Parser p = nxml->parser; + + PARSE_FAIL(p, "parse failed - DOCTYPE not allowed"); +} + struct notification_xml * new_notification_xml(XML_Parser p, struct rrdp_session *repository, - struct rrdp_session *current) + struct rrdp_session *current, const char *notifyuri) { struct notification_xml *nxml; @@ -316,10 +329,13 @@ new_notification_xml(XML_Parser p, struc nxml->parser = p; nxml->repository = repository; nxml->current = current; + nxml->notifyuri = notifyuri; XML_SetElementHandler(nxml->parser, notification_xml_elem_start, notification_xml_elem_end); XML_SetUserData(nxml->parser, nxml); + XML_SetDoctypeDeclHandler(nxml->parser, notification_doctype_handler, + NULL); return nxml; } @@ -351,7 +367,7 @@ enum rrdp_task notification_done(struct notification_xml *nxml, char *last_mod) { struct delta_item *d; - long long s, last_s; + long long s, last_s = 0; nxml->current->last_mod = last_mod; nxml->current->session_id = xstrdup(nxml->session_id); @@ -365,10 +381,15 @@ notification_done(struct notification_xm if (nxml->repository->serial == 0) goto snapshot; - if (nxml->repository->serial == nxml->serial) { - nxml->current->serial = nxml->serial; + /* if our serial is equal or bigger, the repo is up to date */ + if (nxml->repository->serial >= nxml->serial) { + nxml->current->serial = nxml->repository->serial; return NOTIFICATION; } + + /* it makes no sense to process too many deltas */ + if (nxml->serial - nxml->repository->serial > 300) + goto snapshot; /* check that all needed deltas are available */ s = nxml->repository->serial + 1; Index: usr.sbin/rpki-client/rrdp_snapshot.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rrdp_snapshot.c,v retrieving revision 1.1 diff -u -p -u -r1.1 rrdp_snapshot.c --- usr.sbin/rpki-client/rrdp_snapshot.c 1 Apr 2021 16:04:48 -0000 1.1 +++ usr.sbin/rpki-client/rrdp_snapshot.c 6 Nov 2021 18:38:27 -0000 @@ -79,7 +79,7 @@ start_snapshot_elem(struct snapshot_xml } PARSE_FAIL(p, "parse failed - non conforming " - "attribute found in snapshot elem"); + "attribute '%s' found in snapshot elem", attr[i]); } if (!(has_xmlns && sxml->version && sxml->session_id && sxml->serial)) PARSE_FAIL(p, @@ -193,9 +193,21 @@ static void snapshot_content_handler(void *data, const char *content, int length) { struct snapshot_xml *sxml = data; + XML_Parser p = sxml->parser; if (sxml->scope == SNAPSHOT_SCOPE_PUBLISH) - publish_add_content(sxml->pxml, content, length); + if (publish_add_content(sxml->pxml, content, length) == -1) + PARSE_FAIL(p, "parse failed - content too big"); +} + +static void +snapshot_doctype_handler(void *data, const char *doctypeName, + const char *sysid, const char *pubid, int subset) +{ + struct snapshot_xml *sxml = data; + XML_Parser p = sxml->parser; + + PARSE_FAIL(p, "parse failed - DOCTYPE not allowed"); } struct snapshot_xml * @@ -216,6 +228,8 @@ new_snapshot_xml(XML_Parser p, struct rr snapshot_xml_elem_end); XML_SetCharacterDataHandler(sxml->parser, snapshot_content_handler); XML_SetUserData(sxml->parser, sxml); + XML_SetDoctypeDeclHandler(sxml->parser, snapshot_doctype_handler, + NULL); return sxml; } Index: usr.sbin/rpki-client/rsync.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/rsync.c,v retrieving revision 1.23 diff -u -p -u -r1.23 rsync.c --- usr.sbin/rpki-client/rsync.c 1 Apr 2021 11:04:30 -0000 1.23 +++ usr.sbin/rpki-client/rsync.c 6 Nov 2021 18:20:36 -0000 @@ -33,6 +33,9 @@ #include "extern.h" +#define __STRINGIFY(x) #x +#define STRINGIFY(x) __STRINGIFY(x) + /* * A running rsync process. * We can have multiple of these simultaneously and need to keep track @@ -116,17 +119,11 @@ proc_child(int signal) void proc_rsync(char *prog, char *bind_addr, int fd) { - size_t id, i, idsz = 0; - ssize_t ssz; - char *uri = NULL, *dst = NULL, *path, *save, *cmd; - const char *pp; - pid_t pid; - char *args[32]; - int st, rc = 0; - struct stat stt; + size_t i, idsz = 0, nprocs = 0; + int rc = 0; struct pollfd pfd; struct msgbuf msgq; - struct ibuf *b; + struct ibuf *b, *inbuf = NULL; sigset_t mask, oldmask; struct rsyncproc *ids = NULL; @@ -143,6 +140,10 @@ proc_rsync(char *prog, char *bind_addr, */ if (strchr(prog, '/') == NULL) { + const char *pp; + char *save, *cmd, *path; + struct stat stt; + if (getenv("PATH") == NULL) errx(1, "PATH is unset"); if ((path = strdup(getenv("PATH"))) == NULL) @@ -180,7 +181,14 @@ proc_rsync(char *prog, char *bind_addr, err(1, NULL); for (;;) { - pfd.events = POLLIN; + char *uri = NULL, *dst = NULL; + size_t id; + pid_t pid; + int st; + + pfd.events = 0; + if (nprocs < MAX_RSYNC_PROCESSES) + pfd.events |= POLLIN; if (msgq.queued) pfd.events |= POLLOUT; @@ -213,17 +221,16 @@ proc_rsync(char *prog, char *bind_addr, ok = 0; } - b = ibuf_open(sizeof(size_t) + sizeof(ok)); - if (b == NULL) - err(1, NULL); + b = io_new_buffer(); io_simple_buffer(b, &ids[i].id, sizeof(size_t)); io_simple_buffer(b, &ok, sizeof(ok)); - ibuf_close(&msgq, b); + io_close_buffer(&msgq, b); free(ids[i].uri); ids[i].uri = NULL; ids[i].pid = 0; ids[i].id = 0; + nprocs--; } if (pid == -1 && errno != ECHILD) err(1, "waitpid"); @@ -239,23 +246,24 @@ proc_rsync(char *prog, char *bind_addr, } } + /* connection closed */ + if (pfd.revents & POLLHUP) + break; + if (!(pfd.revents & POLLIN)) continue; - /* - * Read til the parent exits. - * That will mean that we can safely exit. - */ - - if ((ssz = read(fd, &id, sizeof(size_t))) == -1) - err(1, "read"); - if (ssz == 0) - break; + b = io_buf_read(fd, &inbuf); + if (b == NULL) + continue; /* Read host and module. */ + io_read_buf(b, &id, sizeof(id)); + io_read_str(b, &dst); + io_read_str(b, &uri); + + ibuf_free(b); - io_str_read(fd, &dst); - io_str_read(fd, &uri); assert(dst); assert(uri); @@ -265,14 +273,23 @@ proc_rsync(char *prog, char *bind_addr, err(1, "fork"); if (pid == 0) { + char *args[32]; + if (pledge("stdio exec", NULL) == -1) err(1, "pledge"); i = 0; args[i++] = (char *)prog; args[i++] = "-rt"; args[i++] = "--no-motd"; - args[i++] = "--timeout"; - args[i++] = "180"; + args[i++] = "--max-size=" STRINGIFY(MAX_FILE_SIZE); + args[i++] = "--timeout=180"; + args[i++] = "--include=*/"; + args[i++] = "--include=*.cer"; + args[i++] = "--include=*.crl"; + args[i++] = "--include=*.gbr"; + args[i++] = "--include=*.mft"; + args[i++] = "--include=*.roa"; + args[i++] = "--exclude=*"; if (bind_addr != NULL) { args[i++] = "--address"; args[i++] = (char *)bind_addr; @@ -280,6 +297,7 @@ proc_rsync(char *prog, char *bind_addr, args[i++] = uri; args[i++] = dst; args[i] = NULL; + /* XXX args overflow not prevented */ execvp(args[0], args); err(1, "%s: execvp", prog); } @@ -299,6 +317,7 @@ proc_rsync(char *prog, char *bind_addr, ids[i].id = id; ids[i].pid = pid; ids[i].uri = uri; + nprocs++; /* Clean up temporary values. */ Index: usr.sbin/rpki-client/tal.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/tal.c,v retrieving revision 1.30 diff -u -p -u -r1.30 tal.c --- usr.sbin/rpki-client/tal.c 1 Apr 2021 06:43:23 -0000 1.30 +++ usr.sbin/rpki-client/tal.c 6 Nov 2021 18:20:20 -0000 @@ -41,7 +41,7 @@ tal_cmp(const void *a, const void *b) * The pointer must be freed with tal_free(). */ static struct tal * -tal_parse_buffer(const char *fn, char *buf) +tal_parse_buffer(const char *fn, char *buf, size_t len) { char *nl, *line, *f, *file = NULL; unsigned char *der; @@ -49,18 +49,33 @@ tal_parse_buffer(const char *fn, char *b int rc = 0; struct tal *tal = NULL; EVP_PKEY *pkey = NULL; + int optcomment = 1; if ((tal = calloc(1, sizeof(struct tal))) == NULL) err(1, NULL); /* Begin with the URI section, comment section already removed. */ - while ((nl = strchr(buf, '\n')) != NULL) { + while ((nl = memchr(buf, '\n', len)) != NULL) { line = buf; - *nl = '\0'; /* advance buffer to next line */ + len -= nl + 1 - buf; buf = nl + 1; + /* replace LF and optional CR with NUL, point nl at first NUL */ + *nl = '\0'; + if (nl > line && nl[-1] == '\r') { + nl[-1] = '\0'; + nl--; + } + + if (optcomment) { + /* if this is a comment, just eat the line */ + if (line[0] == '#') + continue; + optcomment = 0; + } + /* Zero-length line is end of section. */ if (*line == '\0') break; @@ -112,7 +127,7 @@ tal_parse_buffer(const char *fn, char *b qsort(tal->uri, tal->urisz, sizeof(tal->uri[0]), tal_cmp); /* Now the Base64-encoded public key. */ - if ((base64_decode(buf, &der, &dersz)) == -1) { + if ((base64_decode(buf, len, &der, &dersz)) == -1) { warnx("%s: RFC 7730 section 2.1: subjectPublicKeyInfo: " "bad public key", fn); goto out; @@ -144,13 +159,13 @@ out: * Returns the encoded data or NULL on syntax failure. */ struct tal * -tal_parse(const char *fn, char *buf) +tal_parse(const char *fn, char *buf, size_t len) { struct tal *p; const char *d; size_t dlen; - p = tal_parse_buffer(fn, buf); + p = tal_parse_buffer(fn, buf, len); if (p == NULL) return NULL; @@ -170,76 +185,6 @@ tal_parse(const char *fn, char *buf) } /* - * Read the file named "file" into a returned, NUL-terminated buffer. - * This replaces CRLF terminators with plain LF, if found, and also - * elides document-leading comment lines starting with "#". - * Files may not exceeds 4096 bytes. - * This function exits on failure, so it always returns a buffer with - * TAL data. - */ -char * -tal_read_file(const char *file) -{ - char *nbuf, *line = NULL, *buf = NULL; - FILE *in; - ssize_t n, i; - size_t sz = 0, bsz = 0; - int optcomment = 1; - - if ((in = fopen(file, "r")) == NULL) - err(1, "fopen: %s", file); - - while ((n = getline(&line, &sz, in)) != -1) { - /* replace CRLF with just LF */ - if (n > 1 && line[n - 1] == '\n' && line[n - 2] == '\r') { - line[n - 2] = '\n'; - line[n - 1] = '\0'; - n--; - } - if (optcomment) { - /* if this is comment, just eat the line */ - if (line[0] == '#') - continue; - optcomment = 0; - /* - * Empty line is end of section and needs - * to be eaten as well. - */ - if (line[0] == '\n') - continue; - } - - /* make sure every line is valid ascii */ - for (i = 0; i < n; i++) - if (!isprint((unsigned char)line[i]) && - !isspace((unsigned char)line[i])) - errx(1, "getline: %s: " - "invalid content", file); - - /* concat line to buf */ - if ((nbuf = realloc(buf, bsz + n + 1)) == NULL) - err(1, NULL); - if (buf == NULL) - nbuf[0] = '\0'; /* initialize buffer */ - buf = nbuf; - bsz += n + 1; - if (strlcat(buf, line, bsz) >= bsz) - errx(1, "strlcat overflow"); - /* limit the buffer size */ - if (bsz > 4096) - errx(1, "%s: file too big", file); - } - - free(line); - if (ferror(in)) - err(1, "getline: %s", file); - fclose(in); - if (buf == NULL) - errx(1, "%s: no data", file); - return buf; -} - -/* * Free a TAL pointer. * Safe to call with NULL. */ @@ -270,9 +215,10 @@ tal_buffer(struct ibuf *b, const struct { size_t i; + io_simple_buffer(b, &p->id, sizeof(p->id)); io_buf_buffer(b, p->pkey, p->pkeysz); io_str_buffer(b, p->descr); - io_simple_buffer(b, &p->urisz, sizeof(size_t)); + io_simple_buffer(b, &p->urisz, sizeof(p->urisz)); for (i = 0; i < p->urisz; i++) io_str_buffer(b, p->uri[i]); @@ -284,7 +230,7 @@ tal_buffer(struct ibuf *b, const struct * A returned pointer must be freed with tal_free(). */ struct tal * -tal_read(int fd) +tal_read(struct ibuf *b) { size_t i; struct tal *p; @@ -292,18 +238,19 @@ tal_read(int fd) if ((p = calloc(1, sizeof(struct tal))) == NULL) err(1, NULL); - io_buf_read_alloc(fd, (void **)&p->pkey, &p->pkeysz); + io_read_buf(b, &p->id, sizeof(p->id)); + io_read_buf_alloc(b, (void **)&p->pkey, &p->pkeysz); + io_read_str(b, &p->descr); + io_read_buf(b, &p->urisz, sizeof(p->urisz)); assert(p->pkeysz > 0); - io_str_read(fd, &p->descr); assert(p->descr); - io_simple_read(fd, &p->urisz, sizeof(size_t)); assert(p->urisz > 0); if ((p->uri = calloc(p->urisz, sizeof(char *))) == NULL) err(1, NULL); for (i = 0; i < p->urisz; i++) { - io_str_read(fd, &p->uri[i]); + io_read_str(b, &p->uri[i]); assert(p->uri[i]); } Index: usr.sbin/rpki-client/validate.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/validate.c,v retrieving revision 1.13 diff -u -p -u -r1.13 validate.c --- usr.sbin/rpki-client/validate.c 5 Mar 2021 17:15:19 -0000 1.13 +++ usr.sbin/rpki-client/validate.c 6 Nov 2021 18:21:26 -0000 @@ -30,14 +30,6 @@ #include "extern.h" -static void -tracewarn(const struct auth *a) -{ - - for (; a != NULL; a = a->parent) - warnx(" ...inheriting from: %s", a->fn); -} - /* * Walk up the chain of certificates trying to match our AS number to * one of the allocations in that chain. @@ -53,8 +45,7 @@ valid_as(struct auth *a, uint32_t min, u /* Does this certificate cover our AS number? */ if (a->cert->asz) { - c = as_check_covered(min, max, - a->cert->as, a->cert->asz); + c = as_check_covered(min, max, a->cert->as, a->cert->asz); if (c > 0) return 1; else if (c < 0) @@ -81,8 +72,7 @@ valid_ip(struct auth *a, enum afi afi, return 0; /* Does this certificate cover our IP prefix? */ - c = ip_addr_check_covered(afi, min, max, - a->cert->ips, a->cert->ipsz); + c = ip_addr_check_covered(afi, min, max, a->cert->ips, a->cert->ipsz); if (c > 0) return 1; else if (c < 0) @@ -165,8 +155,11 @@ valid_cert(const char *fn, struct auth_t return 0; for (i = 0; i < cert->asz; i++) { - if (cert->as[i].type == CERT_AS_INHERIT) + if (cert->as[i].type == CERT_AS_INHERIT) { + if (cert->purpose == CERT_PURPOSE_BGPSEC_ROUTER) + return 0; /* BGPsec doesn't permit inheriting */ continue; + } min = cert->as[i].type == CERT_AS_ID ? cert->as[i].id : cert->as[i].range.min; max = cert->as[i].type == CERT_AS_ID ? @@ -175,7 +168,6 @@ valid_cert(const char *fn, struct auth_t continue; warnx("%s: RFC 6487: uncovered AS: " "%u--%u", fn, min, max); - tracewarn(a); return 0; } @@ -197,12 +189,12 @@ valid_cert(const char *fn, struct auth_t cert->ips[i].afi, buf1, sizeof(buf1)); warnx("%s: RFC 6487: uncovered IP: " "%s", fn, buf1); + break; case CERT_IP_INHERIT: warnx("%s: RFC 6487: uncovered IP: " "(inherit)", fn); break; } - tracewarn(a); return 0; } @@ -225,8 +217,7 @@ valid_roa(const char *fn, struct auth_tr if (a == NULL) return 0; - if ((roa->tal = strdup(a->tal)) == NULL) - err(1, NULL); + roa->talid = a->cert->talid; for (i = 0; i < roa->ipsz; i++) { if (valid_ip(a, roa->ips[i].afi, roa->ips[i].min, @@ -236,7 +227,6 @@ valid_roa(const char *fn, struct auth_tr roa->ips[i].afi, buf, sizeof(buf)); warnx("%s: RFC 6482: uncovered IP: " "%s", fn, buf); - tracewarn(a); return 0; } @@ -244,6 +234,40 @@ valid_roa(const char *fn, struct auth_tr } /* + * Validate a filename listed on a Manifest. + * draft-ietf-sidrops-6486bis section 4.2.2 + * Returns 1 if filename is valid, otherwise 0. + */ +int +valid_filename(const char *fn) +{ + size_t sz; + const unsigned char *c; + + sz = strlen(fn); + if (sz < 5) + return 0; + + for (c = fn; *c != '\0'; ++c) + if (!isalnum(*c) && *c != '-' && *c != '_' && *c != '.') + return 0; + + if (strchr(fn, '.') != strrchr(fn, '.')) + return 0; + + if (strcasecmp(fn + sz - 4, ".cer") == 0) + return 1; + if (strcasecmp(fn + sz - 4, ".crl") == 0) + return 1; + if (strcasecmp(fn + sz - 4, ".gbr") == 0) + return 1; + if (strcasecmp(fn + sz - 4, ".roa") == 0) + return 1; + + return 0; +} + +/* * Validate a file by verifying the SHA256 hash of that file. * Returns 1 if valid, 0 otherwise. */ @@ -285,6 +309,9 @@ valid_uri(const char *uri, size_t usz, c { size_t s; + if (usz > MAX_URI_LENGTH) + return 0; + for (s = 0; s < usz; s++) if (!isalnum((unsigned char)uri[s]) && !ispunct((unsigned char)uri[s])) @@ -298,6 +325,30 @@ valid_uri(const char *uri, size_t usz, c /* do not allow files or directories to start with a '.' */ if (strstr(uri, "/.") != NULL) + return 0; + + return 1; +} + +/* + * Validate that a URI has the same host as the URI passed in proto. + * Returns 1 if valid, 0 otherwise. + */ +int +valid_origin(const char *uri, const char *proto) +{ + const char *to; + + /* extract end of host from proto URI */ + to = strstr(proto, "://"); + if (to == NULL) + return 0; + to += strlen("://"); + if ((to = strchr(to, '/')) == NULL) + return 0; + + /* compare hosts including the / for the start of the path section */ + if (strncasecmp(uri, proto, to - proto + 1) != 0) return 0; return 1; Index: usr.sbin/rpki-client/version.h =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/version.h,v retrieving revision 1.1 diff -u -p -u -r1.1 version.h --- usr.sbin/rpki-client/version.h 14 Apr 2021 18:05:47 -0000 1.1 +++ usr.sbin/rpki-client/version.h 8 Nov 2021 13:36:57 -0000 @@ -1,3 +1,3 @@ /* $OpenBSD: version.h,v 1.1 2021/04/14 18:05:47 benno Exp $ */ -#define RPKI_VERSION "7.0" +#define RPKI_VERSION "7.5" Index: usr.sbin/rpki-client/x509.c =================================================================== RCS file: /cvs/src/usr.sbin/rpki-client/x509.c,v retrieving revision 1.21 diff -u -p -u -r1.21 x509.c --- usr.sbin/rpki-client/x509.c 1 Apr 2021 06:43:23 -0000 1.21 +++ usr.sbin/rpki-client/x509.c 6 Nov 2021 18:21:12 -0000 @@ -1,5 +1,6 @@ /* $OpenBSD: x509.c,v 1.21 2021/04/01 06:43:23 claudio Exp $ */ /* + * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons * * Permission to use, copy, modify, and distribute this software for any @@ -24,10 +25,20 @@ #include #include +#include #include #include "extern.h" +static ASN1_OBJECT *bgpsec_oid; /* id-kp-bgpsec-router */ + +static void +init_oid(void) +{ + if ((bgpsec_oid = OBJ_txt2obj("1.3.6.1.5.5.7.3.30", 1)) == NULL) + errx(1, "OBJ_txt2obj for %s failed", "1.3.6.1.5.5.7.3.30"); +} + /* * Parse X509v3 authority key identifier (AKI), RFC 6487 sec. 4.8.3. * Returns the AKI or NULL if it could not be parsed. @@ -79,6 +90,7 @@ x509_get_aki(X509 *x, int ta, const char } res = hex_encode(d, dsz); + out: AUTHORITY_KEYID_free(akid); return res; @@ -125,6 +137,109 @@ out: } /* + * Check the certificate's purpose: CA or BGPsec Router. + * Return a member of enum cert_purpose. + */ +enum cert_purpose +x509_get_purpose(X509 *x, const char *fn) +{ + EXTENDED_KEY_USAGE *eku = NULL; + int crit; + enum cert_purpose purpose = 0; + + if (X509_check_ca(x) == 1) { + purpose = CERT_PURPOSE_CA; + goto out; + } + + eku = X509_get_ext_d2i(x, NID_ext_key_usage, &crit, NULL); + if (eku == NULL) { + warnx("%s: EKU: extension missing", fn); + goto out; + } + if (crit != 0) { + warnx("%s: EKU: extension must not be marked critical", fn); + goto out; + } + if (sk_ASN1_OBJECT_num(eku) != 1) { + warnx("%s: EKU: expected 1 purpose, have %d", fn, + sk_ASN1_OBJECT_num(eku)); + goto out; + } + + if (bgpsec_oid == NULL) + init_oid(); + + if (OBJ_cmp(bgpsec_oid, sk_ASN1_OBJECT_value(eku, 0)) == 0) { + purpose = CERT_PURPOSE_BGPSEC_ROUTER; + goto out; + } + + out: + EXTENDED_KEY_USAGE_free(eku); + return purpose; +} + +/* + * Extract Subject Public Key Info (SPKI) from BGPsec X.509 Certificate. + * Returns NULL on failure, on success return the SPKI as base64 encoded pubkey + */ +char * +x509_get_pubkey(X509 *x, const char *fn) +{ + EVP_PKEY *pkey; + EC_KEY *eckey; + int nid; + const char *cname; + uint8_t *pubkey = NULL; + char *res = NULL; + int len; + + pkey = X509_get0_pubkey(x); + if (pkey == NULL) { + warnx("%s: X509_get_pubkey failed in %s", fn, __func__); + goto out; + } + if (EVP_PKEY_base_id(pkey) != EVP_PKEY_EC) { + warnx("%s: Expected EVP_PKEY_EC, got %d", fn, + EVP_PKEY_base_id(pkey)); + goto out; + } + + eckey = EVP_PKEY_get0_EC_KEY(pkey); + if (eckey == NULL) { + warnx("%s: Incorrect key type", fn); + goto out; + } + + nid = EC_GROUP_get_curve_name(EC_KEY_get0_group(eckey)); + if (nid != NID_X9_62_prime256v1) { + if ((cname = EC_curve_nid2nist(nid)) == NULL) + cname = OBJ_nid2sn(nid); + warnx("%s: Expected P-256, got %s", fn, cname); + goto out; + } + + if (!EC_KEY_check_key(eckey)) { + warnx("%s: EC_KEY_check_key failed in %s", fn, __func__); + goto out; + } + + len = i2d_PUBKEY(pkey, &pubkey); + if (len <= 0) { + warnx("%s: i2d_PUBKEY failed in %s", fn, __func__); + goto out; + } + + if (base64_encode(pubkey, len, &res) == -1) + errx(1, "base64_encode failed in %s", __func__); + + out: + free(pubkey); + return res; +} + +/* * Parse the Authority Information Access (AIA) extension * See RFC 6487, section 4.8.7 for details. * Returns NULL on failure, on success returns the AIA URI @@ -167,6 +282,13 @@ x509_get_aia(X509 *x, const char *fn) goto out; } + if (ASN1_STRING_length(ad->location->d.uniformResourceIdentifier) + > MAX_URI_LENGTH) { + warnx("%s: RFC 6487 section 4.8.7: AIA: " + "URI exceeds max length of %d", fn, MAX_URI_LENGTH); + goto out; + } + aia = strndup( ASN1_STRING_get0_data(ad->location->d.uniformResourceIdentifier), ASN1_STRING_length(ad->location->d.uniformResourceIdentifier)); @@ -179,6 +301,34 @@ out: } /* + * Extract the expire time (not-after) of a certificate. + */ +int +x509_get_expire(X509 *x, const char *fn, time_t *tt) +{ + const ASN1_TIME *at; + struct tm expires_tm; + time_t expires; + + at = X509_get0_notAfter(x); + if (at == NULL) { + warnx("%s: X509_get0_notafter failed", fn); + return 0; + } + memset(&expires_tm, 0, sizeof(expires_tm)); + if (ASN1_time_parse(at->data, at->length, &expires_tm, 0) == -1) { + warnx("%s: ASN1_time_parse failed", fn); + return 0; + } + if ((expires = mktime(&expires_tm)) == -1) + errx(1, "%s: mktime failed", fn); + + *tt = expires; + return 1; + +} + +/* * Parse the very specific subset of information in the CRL distribution * point extension. * See RFC 6487, sectoin 4.8.6 for details. @@ -236,6 +386,13 @@ x509_get_crl(X509 *x, const char *fn) if (name->type != GEN_URI) { warnx("%s: RFC 6487 section 4.8.6: CRL: " "want URI type, have %d", fn, name->type); + goto out; + } + + if (ASN1_STRING_length(name->d.uniformResourceIdentifier) + > MAX_URI_LENGTH) { + warnx("%s: RFC 6487 section 4.8.6: CRL: " + "URI exceeds max length of %d", fn, MAX_URI_LENGTH); goto out; } Index: usr.bin/rsync/Makefile =================================================================== RCS file: /cvs/src/usr.bin/rsync/Makefile,v retrieving revision 1.10 diff -u -p -u -r1.10 Makefile --- usr.bin/rsync/Makefile 8 May 2019 21:30:11 -0000 1.10 +++ usr.bin/rsync/Makefile 6 Nov 2021 18:27:23 -0000 @@ -1,14 +1,18 @@ # $OpenBSD: Makefile,v 1.10 2019/05/08 21:30:11 benno Exp $ PROG= openrsync -SRCS= blocks.c client.c downloader.c fargs.c flist.c hash.c ids.c \ - io.c log.c mkpath.c mktemp.c receiver.c sender.c server.c session.c \ - socket.c symlinks.c uploader.c main.c misc.c -LDADD+= -lcrypto -lm -DPADD+= ${LIBCRYPTO} ${LIBM} +SRCS= blocks.c client.c copy.c downloader.c fargs.c flist.c hash.c ids.c \ + io.c log.c main.c misc.c mkpath.c mktemp.c receiver.c rmatch.c \ + rules.c sender.c server.c session.c socket.c symlinks.c uploader.c +LDADD+= -lcrypto -lm -lutil +DPADD+= ${LIBCRYPTO} ${LIBM} ${LIBUTIL} MAN= openrsync.1 -CFLAGS+=-g -W -Wall -Wextra +CFLAGS+= -Wall -Wextra +CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes +CFLAGS+= -Wmissing-declarations +CFLAGS+= -Wshadow + openrsync.1: rsync.1 ln -sf ${.CURDIR}/rsync.1 openrsync.1 Index: usr.bin/rsync/blocks.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/blocks.c,v retrieving revision 1.19 diff -u -p -u -r1.19 blocks.c --- usr.bin/rsync/blocks.c 2 Jun 2019 17:43:34 -0000 1.19 +++ usr.bin/rsync/blocks.c 6 Nov 2021 18:28:34 -0000 @@ -331,10 +331,10 @@ blk_recv_ack(char buf[20], const struct size_t pos = 0, sz; sz = sizeof(int32_t) + /* index */ - sizeof(int32_t) + /* block count */ - sizeof(int32_t) + /* block length */ - sizeof(int32_t) + /* checksum length */ - sizeof(int32_t); /* block remainder */ + sizeof(int32_t) + /* block count */ + sizeof(int32_t) + /* block length */ + sizeof(int32_t) + /* checksum length */ + sizeof(int32_t); /* block remainder */ assert(sz == 20); io_buffer_int(buf, &pos, sz, idx); @@ -457,9 +457,9 @@ blk_send_ack(struct sess *sess, int fd, /* Put the entire send routine into a buffer. */ sz = sizeof(int32_t) + /* block count */ - sizeof(int32_t) + /* block length */ - sizeof(int32_t) + /* checksum length */ - sizeof(int32_t); /* block remainder */ + sizeof(int32_t) + /* block length */ + sizeof(int32_t) + /* checksum length */ + sizeof(int32_t); /* block remainder */ assert(sz <= sizeof(buf)); if (!io_read_buf(sess, fd, buf, sz)) { Index: usr.bin/rsync/charclass.h =================================================================== RCS file: usr.bin/rsync/charclass.h diff -N usr.bin/rsync/charclass.h --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ usr.bin/rsync/charclass.h 6 Nov 2021 14:21:39 -0000 @@ -0,0 +1,29 @@ +/* + * Public domain, 2008, Todd C. Miller + * + * $OpenBSD: charclass.h,v 1.1 2021/08/29 13:43:46 claudio Exp $ + */ + +/* + * POSIX character class support for fnmatch() and glob(). + */ +static const struct cclass { + const char *name; + int (*isctype)(int); +} cclasses[] = { + { "alnum", isalnum }, + { "alpha", isalpha }, + { "blank", isblank }, + { "cntrl", iscntrl }, + { "digit", isdigit }, + { "graph", isgraph }, + { "lower", islower }, + { "print", isprint }, + { "punct", ispunct }, + { "space", isspace }, + { "upper", isupper }, + { "xdigit", isxdigit }, + { NULL, NULL } +}; + +#define NCCLASSES (sizeof(cclasses) / sizeof(cclasses[0]) - 1) Index: usr.bin/rsync/client.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/client.c,v retrieving revision 1.15 diff -u -p -u -r1.15 client.c --- usr.bin/rsync/client.c 8 May 2019 20:00:25 -0000 1.15 +++ usr.bin/rsync/client.c 6 Nov 2021 18:28:53 -0000 @@ -43,7 +43,7 @@ rsync_client(const struct opts *opts, in if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw unveil", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); memset(&sess, 0, sizeof(struct sess)); sess.opts = opts; Index: usr.bin/rsync/copy.c =================================================================== RCS file: usr.bin/rsync/copy.c diff -N usr.bin/rsync/copy.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ usr.bin/rsync/copy.c 6 Nov 2021 14:21:39 -0000 @@ -0,0 +1,90 @@ +/* $OpenBSD: copy.c,v 1.2 2021/10/24 21:24:17 deraadt Exp $ */ +/* + * Copyright (c) 2021 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include /* MAXBSIZE */ + +#include +#include +#include + +#include "extern.h" + +/* + * Return true if all bytes in buffer are zero. + * A buffer of zero lenght is also considered a zero buffer. + */ +static int +iszero(const void *b, size_t len) +{ + const unsigned char *c = b; + + for (; len > 0; len--) { + if (*c++ != '\0') + return 0; + } + return 1; +} + +static int +copy_internal(int fromfd, int tofd) +{ + char buf[MAXBSIZE]; + ssize_t r, w; + + while ((r = read(fromfd, buf, sizeof(buf))) > 0) { + if (iszero(buf, sizeof(buf))) { + if (lseek(tofd, r, SEEK_CUR) == -1) + return -1; + } else { + w = write(tofd, buf, r); + if (r != w || w == -1) + return -1; + } + } + if (r == -1) + return -1; + if (ftruncate(tofd, lseek(tofd, 0, SEEK_CUR)) == -1) + return -1; + return 0; +} + +void +copy_file(int rootfd, const char *basedir, const struct flist *f) +{ + int fromfd, tofd, dfd; + + dfd = openat(rootfd, basedir, O_RDONLY | O_DIRECTORY); + if (dfd == -1) + err(ERR_FILE_IO, "%s: openat", basedir); + + fromfd = openat(dfd, f->path, O_RDONLY | O_NOFOLLOW); + if (fromfd == -1) + err(ERR_FILE_IO, "%s/%s: openat", basedir, f->path); + close(dfd); + + tofd = openat(rootfd, f->path, + O_WRONLY | O_NOFOLLOW | O_TRUNC | O_CREAT | O_EXCL, + 0600); + if (tofd == -1) + err(ERR_FILE_IO, "%s: openat", f->path); + + if (copy_internal(fromfd, tofd) == -1) + err(ERR_FILE_IO, "%s: copy file", f->path); + + close(fromfd); + close(tofd); +} Index: usr.bin/rsync/downloader.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/downloader.c,v retrieving revision 1.21 diff -u -p -u -r1.21 downloader.c --- usr.bin/rsync/downloader.c 8 May 2019 21:30:11 -0000 1.21 +++ usr.bin/rsync/downloader.c 6 Nov 2021 18:33:43 -0000 @@ -350,7 +350,7 @@ rsync_downloader(struct download *p, str p->state = DOWNLOAD_READ_LOCAL; f = &p->fl[idx]; - p->ofd = openat(p->rootfd, f->path, O_RDONLY | O_NONBLOCK, 0); + p->ofd = openat(p->rootfd, f->path, O_RDONLY | O_NONBLOCK); if (p->ofd == -1 && errno != ENOENT) { ERR("%s: openat", f->path); Index: usr.bin/rsync/extern.h =================================================================== RCS file: /cvs/src/usr.bin/rsync/extern.h,v retrieving revision 1.36 diff -u -p -u -r1.36 extern.h --- usr.bin/rsync/extern.h 31 Mar 2021 19:45:16 -0000 1.36 +++ usr.bin/rsync/extern.h 6 Nov 2021 18:31:51 -0000 @@ -34,6 +34,15 @@ #define BLOCK_SIZE_MIN (700) /* + * Maximum number of base directories that can be used. + */ +#define MAX_BASEDIR 20 + +#define BASE_MODE_COMPARE 1 +#define BASE_MODE_COPY 2 +#define BASE_MODE_LINK 3 + +/* * The sender and receiver use a two-phase synchronisation process. * The first uses two-byte hashes; the second, 16-byte. * (The second must hold a full MD4 digest.) @@ -42,6 +51,19 @@ #define CSUM_LENGTH_PHASE2 (16) /* + * Rsync error codes. + */ +#define ERR_SYNTAX 1 +#define ERR_PROTOCOL 2 +#define ERR_SOCK_IO 10 +#define ERR_FILE_IO 11 +#define ERR_WIREPROTO 12 +#define ERR_IPC 14 /* catchall for any kind of syscall error */ +#define ERR_TERMIMATED 16 +#define ERR_WAITPID 21 +#define ERR_NOMEM 22 + +/* * Use this for --timeout. * All poll events will use it and catch time-outs. */ @@ -118,10 +140,29 @@ struct opts { int no_motd; /* --no-motd */ int numeric_ids; /* --numeric-ids */ int one_file_system; /* -x */ + int alt_base_mode; + off_t max_size; /* --max-size */ + off_t min_size; /* --min-size */ char *rsync_path; /* --rsync-path */ char *ssh_prog; /* --rsh or -e */ char *port; /* --port */ char *address; /* --address */ + char *basedir[MAX_BASEDIR]; +}; + +enum rule_type { + RULE_NONE, + RULE_EXCLUDE, + RULE_INCLUDE, + RULE_CLEAR, +#ifdef NOTYET + RULE_MERGE, + RULE_DIR_MERGE, + RULE_SHOW, + RULE_HIDE, + RULE_PROTECT, + RULE_RISK, +#endif }; /* @@ -246,134 +287,134 @@ extern int verbose; #define ERRX(_fmt, ...) \ rsync_errx( (_fmt), ##__VA_ARGS__) -void rsync_log(int, const char *, ...) +void rsync_log(int, const char *, ...) __attribute__((format(printf, 2, 3))); -void rsync_warnx1(const char *, ...) +void rsync_warnx1(const char *, ...) __attribute__((format(printf, 1, 2))); -void rsync_warn(int, const char *, ...) +void rsync_warn(int, const char *, ...) __attribute__((format(printf, 2, 3))); -void rsync_warnx(const char *, ...) +void rsync_warnx(const char *, ...) __attribute__((format(printf, 1, 2))); -void rsync_err(const char *, ...) +void rsync_err(const char *, ...) __attribute__((format(printf, 1, 2))); -void rsync_errx(const char *, ...) +void rsync_errx(const char *, ...) __attribute__((format(printf, 1, 2))); -void rsync_errx1(const char *, ...) +void rsync_errx1(const char *, ...) __attribute__((format(printf, 1, 2))); -int flist_del(struct sess *, int, - const struct flist *, size_t); -int flist_gen(struct sess *, size_t, char **, - struct flist **, size_t *); -int flist_gen_local(struct sess *, const char *, - struct flist **, size_t *); -void flist_free(struct flist *, size_t); -int flist_recv(struct sess *, int, - struct flist **, size_t *); -int flist_send(struct sess *, int, int, - const struct flist *, size_t); -int flist_gen_dels(struct sess *, const char *, - struct flist **, size_t *, - const struct flist *, size_t); +int flist_del(struct sess *, int, const struct flist *, size_t); +int flist_gen(struct sess *, size_t, char **, struct flist **, size_t *); +int flist_gen_local(struct sess *, const char *, struct flist **, size_t *); +void flist_free(struct flist *, size_t); +int flist_recv(struct sess *, int, struct flist **, size_t *); +int flist_send(struct sess *, int, int, const struct flist *, size_t); +int flist_gen_dels(struct sess *, const char *, struct flist **, size_t *, + const struct flist *, size_t); +const char *alt_base_mode(int); char **fargs_cmdline(struct sess *, const struct fargs *, size_t *); -int io_read_buf(struct sess *, int, void *, size_t); -int io_read_byte(struct sess *, int, uint8_t *); -int io_read_check(int); -int io_read_flush(struct sess *, int); -int io_read_int(struct sess *, int, int32_t *); -int io_read_uint(struct sess *, int, uint32_t *); -int io_read_long(struct sess *, int, int64_t *); -int io_read_size(struct sess *, int, size_t *); -int io_read_ulong(struct sess *, int, uint64_t *); -int io_write_buf(struct sess *, int, const void *, size_t); -int io_write_byte(struct sess *, int, uint8_t); -int io_write_int(struct sess *, int, int32_t); -int io_write_uint(struct sess *, int, uint32_t); -int io_write_line(struct sess *, int, const char *); -int io_write_long(struct sess *, int, int64_t); -int io_write_ulong(struct sess *, int, uint64_t); - -int io_lowbuffer_alloc(struct sess *, void **, - size_t *, size_t *, size_t); -void io_lowbuffer_int(struct sess *, void *, - size_t *, size_t, int32_t); -void io_lowbuffer_buf(struct sess *, void *, - size_t *, size_t, const void *, size_t); - -void io_buffer_int(void *, size_t *, size_t, int32_t); -void io_buffer_buf(void *, size_t *, size_t, const void *, size_t); - -void io_unbuffer_int(const void *, - size_t *, size_t, int32_t *); -int io_unbuffer_size(const void *, size_t *, size_t, size_t *); -void io_unbuffer_buf(const void *, size_t *, size_t, void *, size_t); - -int rsync_receiver(struct sess *, int, int, const char *); -int rsync_sender(struct sess *, int, int, size_t, char **); -int rsync_client(const struct opts *, int, const struct fargs *); -int rsync_connect(const struct opts *, int *, - const struct fargs *); -int rsync_socket(const struct opts *, int, const struct fargs *); -int rsync_server(const struct opts *, size_t, char *[]); -int rsync_downloader(struct download *, struct sess *, int *); -int rsync_set_metadata(struct sess *, int, int, - const struct flist *, const char *); -int rsync_set_metadata_at(struct sess *, int, int, - const struct flist *, const char *); -int rsync_uploader(struct upload *, - int *, struct sess *, int *); -int rsync_uploader_tail(struct upload *, struct sess *); - -struct download *download_alloc(struct sess *, int, - const struct flist *, size_t, int); -void download_free(struct download *); -struct upload *upload_alloc(const char *, int, int, size_t, - const struct flist *, size_t, mode_t); -void upload_free(struct upload *); +int io_read_buf(struct sess *, int, void *, size_t); +int io_read_byte(struct sess *, int, uint8_t *); +int io_read_check(int); +int io_read_flush(struct sess *, int); +int io_read_int(struct sess *, int, int32_t *); +int io_read_uint(struct sess *, int, uint32_t *); +int io_read_long(struct sess *, int, int64_t *); +int io_read_size(struct sess *, int, size_t *); +int io_read_ulong(struct sess *, int, uint64_t *); +int io_write_buf(struct sess *, int, const void *, size_t); +int io_write_byte(struct sess *, int, uint8_t); +int io_write_int(struct sess *, int, int32_t); +int io_write_uint(struct sess *, int, uint32_t); +int io_write_line(struct sess *, int, const char *); +int io_write_long(struct sess *, int, int64_t); +int io_write_ulong(struct sess *, int, uint64_t); + +int io_lowbuffer_alloc(struct sess *, void **, size_t *, size_t *, size_t); +void io_lowbuffer_int(struct sess *, void *, size_t *, size_t, int32_t); +void io_lowbuffer_buf(struct sess *, void *, size_t *, size_t, const void *, + size_t); + +void io_buffer_int(void *, size_t *, size_t, int32_t); +void io_buffer_buf(void *, size_t *, size_t, const void *, size_t); + +void io_unbuffer_int(const void *, size_t *, size_t, int32_t *); +int io_unbuffer_size(const void *, size_t *, size_t, size_t *); +void io_unbuffer_buf(const void *, size_t *, size_t, void *, size_t); + +int rsync_receiver(struct sess *, int, int, const char *); +int rsync_sender(struct sess *, int, int, size_t, char **); +int rsync_client(const struct opts *, int, const struct fargs *); +int rsync_connect(const struct opts *, int *, const struct fargs *); +int rsync_socket(const struct opts *, int, const struct fargs *); +int rsync_server(const struct opts *, size_t, char *[]); +int rsync_downloader(struct download *, struct sess *, int *); +int rsync_set_metadata(struct sess *, int, int, const struct flist *, + const char *); +int rsync_set_metadata_at(struct sess *, int, int, const struct flist *, + const char *); +int rsync_uploader(struct upload *, int *, struct sess *, int *); +int rsync_uploader_tail(struct upload *, struct sess *); + +struct download *download_alloc(struct sess *, int, const struct flist *, + size_t, int); +void download_free(struct download *); +struct upload *upload_alloc(const char *, int, int, size_t, + const struct flist *, size_t, mode_t); +void upload_free(struct upload *); struct blktab *blkhash_alloc(void); int blkhash_set(struct blktab *, const struct blkset *); void blkhash_free(struct blktab *); -struct blkset *blk_recv(struct sess *, int, const char *); -void blk_recv_ack(char [20], const struct blkset *, int32_t); -void blk_match(struct sess *, const struct blkset *, - const char *, struct blkstat *); -int blk_send(struct sess *, int, size_t, - const struct blkset *, const char *); -int blk_send_ack(struct sess *, int, struct blkset *); - -uint32_t hash_fast(const void *, size_t); -void hash_slow(const void *, size_t, - unsigned char *, const struct sess *); -void hash_file(const void *, size_t, - unsigned char *, const struct sess *); - -int mkpath(char *); - -int mkstempat(int, char *); -char *mkstemplinkat(char*, int, char *); -char *mkstempfifoat(int, char *); -char *mkstempnodat(int, char *, mode_t, dev_t); -char *mkstempsock(const char *, char *); -int mktemplate(char **, const char *, int); - -char *symlink_read(const char *); -char *symlinkat_read(int, const char *); - -int sess_stats_send(struct sess *, int); -int sess_stats_recv(struct sess *, int); - -int idents_add(int, struct ident **, size_t *, int32_t); -void idents_assign_gid(struct sess *, - struct flist *, size_t, const struct ident *, size_t); -void idents_assign_uid(struct sess *, - struct flist *, size_t, const struct ident *, size_t); -void idents_free(struct ident *, size_t); -int idents_recv(struct sess *, int, struct ident **, size_t *); -void idents_remap(struct sess *, int, struct ident *, size_t); -int idents_send(struct sess *, int, const struct ident *, size_t); +struct blkset *blk_recv(struct sess *, int, const char *); +void blk_recv_ack(char [20], const struct blkset *, int32_t); +void blk_match(struct sess *, const struct blkset *, + const char *, struct blkstat *); +int blk_send(struct sess *, int, size_t, const struct blkset *, + const char *); +int blk_send_ack(struct sess *, int, struct blkset *); + +uint32_t hash_fast(const void *, size_t); +void hash_slow(const void *, size_t, unsigned char *, + const struct sess *); +void hash_file(const void *, size_t, unsigned char *, + const struct sess *); + +void copy_file(int, const char *, const struct flist *); + +int mkpath(char *); + +int mkstempat(int, char *); +char *mkstemplinkat(char*, int, char *); +char *mkstempfifoat(int, char *); +char *mkstempnodat(int, char *, mode_t, dev_t); +char *mkstempsock(const char *, char *); +int mktemplate(char **, const char *, int); + +int parse_rule(char *line, enum rule_type); +void parse_file(const char *, enum rule_type); +void send_rules(struct sess *, int); +void recv_rules(struct sess *, int); +int rules_match(const char *, int); + +int rmatch(const char *, const char *, int); + +char *symlink_read(const char *); +char *symlinkat_read(int, const char *); + +int sess_stats_send(struct sess *, int); +int sess_stats_recv(struct sess *, int); + +int idents_add(int, struct ident **, size_t *, int32_t); +void idents_assign_gid(struct sess *, struct flist *, size_t, + const struct ident *, size_t); +void idents_assign_uid(struct sess *, struct flist *, size_t, + const struct ident *, size_t); +void idents_free(struct ident *, size_t); +int idents_recv(struct sess *, int, struct ident **, size_t *); +void idents_remap(struct sess *, int, struct ident *, size_t); +int idents_send(struct sess *, int, const struct ident *, size_t); #endif /*!EXTERN_H*/ Index: usr.bin/rsync/fargs.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/fargs.c,v retrieving revision 1.17 diff -u -p -u -r1.17 fargs.c --- usr.bin/rsync/fargs.c 8 May 2019 20:00:25 -0000 1.17 +++ usr.bin/rsync/fargs.c 6 Nov 2021 18:34:00 -0000 @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,21 @@ #define RSYNC_PATH "rsync" +const char * +alt_base_mode(int mode) +{ + switch (mode) { + case BASE_MODE_COMPARE: + return "--compare-dest"; + case BASE_MODE_COPY: + return "--copy-dest"; + case BASE_MODE_LINK: + return "--link-dest"; + default: + errx(1, "unknown base mode %d", mode); + } +} + char ** fargs_cmdline(struct sess *sess, const struct fargs *f, size_t *skip) { @@ -51,7 +67,7 @@ fargs_cmdline(struct sess *sess, const s if (sess->opts->ssh_prog) { ap = strdup(sess->opts->ssh_prog); if (ap == NULL) - goto out; + err(ERR_NOMEM, NULL); while ((arg = strsep(&ap, " \t")) != NULL) { if (arg[0] == '\0') { @@ -115,6 +131,22 @@ fargs_cmdline(struct sess *sess, const s if (!sess->opts->specials && sess->opts->devices) /* --devices is sent as -D --no-specials */ addargs(&args, "--no-specials"); + if (sess->opts->max_size >= 0) + addargs(&args, "--max-size=%lld", sess->opts->max_size); + if (sess->opts->min_size >= 0) + addargs(&args, "--min-size=%lld", sess->opts->min_size); + + /* only add --compare-dest, etc if this is the sender */ + if (sess->opts->alt_base_mode != 0 && + f->mode == FARGS_SENDER) { + for (j = 0; j < MAX_BASEDIR; j++) { + if (sess->opts->basedir[j] == NULL) + break; + addargs(&args, "%s=%s", + alt_base_mode(sess->opts->alt_base_mode), + sess->opts->basedir[j]); + } + } /* Terminate with a full-stop for reasons unknown. */ @@ -127,8 +159,4 @@ fargs_cmdline(struct sess *sess, const s addargs(&args, "%s", f->sink); return args.list; -out: - freeargs(&args); - ERR("calloc"); - return NULL; } Index: usr.bin/rsync/flist.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/flist.c,v retrieving revision 1.31 diff -u -p -u -r1.31 flist.c --- usr.bin/rsync/flist.c 22 Mar 2021 11:49:15 -0000 1.31 +++ usr.bin/rsync/flist.c 6 Nov 2021 18:34:12 -0000 @@ -15,13 +15,13 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include #include #include #include #include #include +#include #include #include #include @@ -283,7 +283,7 @@ flist_send(struct sess *sess, int fdin, if (sess->mplex_reads && io_read_check(fdin) && - !io_read_flush(sess, fdin)) { + !io_read_flush(sess, fdin)) { ERRX1("io_read_flush"); goto out; } @@ -356,7 +356,7 @@ flist_send(struct sess *sess, int fdin, /* Conditional part: devices & special files. */ if ((sess->opts->devices && (S_ISBLK(f->st.mode) || - S_ISCHR(f->st.mode))) || + S_ISCHR(f->st.mode))) || (sess->opts->specials && (S_ISFIFO(f->st.mode) || S_ISSOCK(f->st.mode)))) { if (!io_write_int(sess, fdout, f->st.rdev)) { @@ -428,7 +428,7 @@ out: */ static int flist_recv_name(struct sess *sess, int fd, struct flist *f, uint8_t flags, - char last[MAXPATHLEN]) + char last[PATH_MAX]) { uint8_t bval; size_t partial = 0; @@ -504,7 +504,7 @@ flist_recv_name(struct sess *sess, int f /* Record our last path and construct our filename. */ - strlcpy(last, f->path, MAXPATHLEN); + strlcpy(last, f->path, PATH_MAX); f->wpath = f->path; return 1; } @@ -593,7 +593,7 @@ flist_recv(struct sess *sess, int fd, st const struct flist *fflast = NULL; size_t flsz = 0, flmax = 0, lsz, gidsz = 0, uidsz = 0; uint8_t flag; - char last[MAXPATHLEN]; + char last[PATH_MAX]; int64_t lval; /* temporary values... */ int32_t ival; uint32_t uival; @@ -694,7 +694,7 @@ flist_recv(struct sess *sess, int fd, st /* Conditional part: devices & special files. */ if ((sess->opts->devices && (S_ISBLK(ff->st.mode) || - S_ISCHR(ff->st.mode))) || + S_ISCHR(ff->st.mode))) || (sess->opts->specials && (S_ISFIFO(ff->st.mode) || S_ISSOCK(ff->st.mode)))) { if (!(FLIST_RDEV_SAME & flag)) { @@ -823,6 +823,11 @@ flist_gen_dirent(struct sess *sess, char ERR("%s: lstat", root); return 0; } else if (S_ISREG(st.st_mode)) { + /* filter files */ + if (rules_match(root, 0) == -1) { + WARNX("%s: skipping excluded file", root); + return 1; + } if (!flist_realloc(fl, sz, max)) { ERRX1("flist_realloc"); return 0; @@ -839,7 +844,13 @@ flist_gen_dirent(struct sess *sess, char if (!sess->opts->preserve_links) { WARNX("%s: skipping symlink", root); return 1; - } else if (!flist_realloc(fl, sz, max)) { + } + /* filter files */ + if (rules_match(root, 0) == -1) { + WARNX("%s: skipping excluded symlink", root); + return 1; + } + if (!flist_realloc(fl, sz, max)) { ERRX1("flist_realloc"); return 0; } @@ -942,6 +953,15 @@ flist_gen_dirent(struct sess *sess, char nxdev++; } + /* filter files */ + if (rules_match(ent->fts_path + stripdir, + (ent->fts_info == FTS_D)) == -1) { + WARNX("%s: skipping excluded file", + ent->fts_path + stripdir); + fts_set(fts, ent, FTS_SKIP); + continue; + } + /* Allocate a new file entry. */ if (!flist_realloc(fl, sz, max)) { @@ -972,7 +992,7 @@ flist_gen_dirent(struct sess *sess, char /* Optionally copy link information. */ if (S_ISLNK(ent->fts_statp->st_mode)) { - f->link = symlink_read(f->path); + f->link = symlink_read(ent->fts_accpath); if (f->link == NULL) { ERRX1("symlink_read"); goto out; @@ -1073,6 +1093,11 @@ flist_gen_files(struct sess *sess, size_ continue; } + /* filter files */ + if (rules_match(argv[i], S_ISDIR(st.st_mode)) == -1) { + WARNX("%s: skipping excluded file", argv[i]); + continue; + } f = &fl[flsz++]; assert(f != NULL); @@ -1295,6 +1320,16 @@ flist_gen_dels(struct sess *sess, const } if (!flag) continue; + } + + /* filter files on delete */ + /* TODO handle --delete-excluded */ + if (rules_match(ent->fts_path + stripdir, + (ent->fts_info == FTS_D)) == -1) { + WARNX("skip excluded file %s", + ent->fts_path + stripdir); + fts_set(fts, ent, FTS_SKIP); + continue; } /* Look up in hashtable. */ Index: usr.bin/rsync/main.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/main.c,v retrieving revision 1.53 diff -u -p -u -r1.53 main.c --- usr.bin/rsync/main.c 31 Mar 2021 19:45:16 -0000 1.53 +++ usr.bin/rsync/main.c 6 Nov 2021 18:35:29 -0000 @@ -26,6 +26,7 @@ #include #include #include +#include #include "extern.h" @@ -83,18 +84,18 @@ fargs_parse(size_t argc, char *argv[], s /* Allocations. */ if ((f = calloc(1, sizeof(struct fargs))) == NULL) - err(1, "calloc"); + err(ERR_NOMEM, NULL); f->sourcesz = argc - 1; if ((f->sources = calloc(f->sourcesz, sizeof(char *))) == NULL) - err(1, "calloc"); + err(ERR_NOMEM, NULL); for (i = 0; i < argc - 1; i++) if ((f->sources[i] = strdup(argv[i])) == NULL) - err(1, "strdup"); + err(ERR_NOMEM, NULL); if ((f->sink = strdup(argv[i])) == NULL) - err(1, "strdup"); + err(ERR_NOMEM, NULL); /* * Test files for its locality. @@ -109,15 +110,16 @@ fargs_parse(size_t argc, char *argv[], s if (fargs_is_remote(f->sink)) { f->mode = FARGS_SENDER; if ((f->host = strdup(f->sink)) == NULL) - err(1, "strdup"); + err(ERR_NOMEM, NULL); } if (fargs_is_remote(f->sources[0])) { if (f->host != NULL) - errx(1, "both source and destination cannot be remote files"); + errx(ERR_SYNTAX, "both source and destination " + "cannot be remote files"); f->mode = FARGS_RECEIVER; if ((f->host = strdup(f->sources[0])) == NULL) - err(1, "strdup"); + err(ERR_NOMEM, NULL); } if (f->host != NULL) { @@ -127,7 +129,8 @@ fargs_parse(size_t argc, char *argv[], s len = strlen(f->host) - 8 + 1; memmove(f->host, f->host + 8, len); if ((cp = strchr(f->host, '/')) == NULL) - errx(1, "rsync protocol requires a module name"); + errx(ERR_SYNTAX, + "rsync protocol requires a module name"); *cp++ = '\0'; f->module = cp; if ((cp = strchr(f->module, '/')) != NULL) @@ -152,9 +155,9 @@ fargs_parse(size_t argc, char *argv[], s } } if ((len = strlen(f->host)) == 0) - errx(1, "empty remote host"); + errx(ERR_SYNTAX, "empty remote host"); if (f->remote && strlen(f->module) == 0) - errx(1, "empty remote module"); + errx(ERR_SYNTAX, "empty remote module"); } /* Make sure we have the same "hostspec" for all files. */ @@ -164,7 +167,7 @@ fargs_parse(size_t argc, char *argv[], s for (i = 0; i < f->sourcesz; i++) { if (!fargs_is_remote(f->sources[i])) continue; - errx(1, + errx(ERR_SYNTAX, "remote file in list of local sources: %s", f->sources[i]); } @@ -174,20 +177,20 @@ fargs_parse(size_t argc, char *argv[], s !fargs_is_daemon(f->sources[i])) continue; if (fargs_is_daemon(f->sources[i])) - errx(1, "remote daemon in list of " - "remote sources: %s", - f->sources[i]); - errx(1, "local file in list of remote sources: %s", - f->sources[i]); + errx(ERR_SYNTAX, + "remote daemon in list of remote " + "sources: %s", f->sources[i]); + errx(ERR_SYNTAX, "local file in list of " + "remote sources: %s", f->sources[i]); } } else { if (f->mode != FARGS_RECEIVER) - errx(1, "sender mode for remote " + errx(ERR_SYNTAX, "sender mode for remote " "daemon receivers not yet supported"); for (i = 0; i < f->sourcesz; i++) { if (fargs_is_daemon(f->sources[i])) continue; - errx(1, "non-remote daemon file " + errx(ERR_SYNTAX, "non-remote daemon file " "in list of remote daemon sources: " "%s", f->sources[i]); } @@ -233,7 +236,7 @@ fargs_parse(size_t argc, char *argv[], s *ccp = '\0'; if (strncmp(cp, f->host, len) || (cp[len] != '/' && cp[len] != '\0')) - errx(1, "different remote host: %s", + errx(ERR_SYNTAX, "different remote host: %s", f->sources[i]); memmove(f->sources[i], f->sources[i] + len + 8 + 1, @@ -246,7 +249,7 @@ fargs_parse(size_t argc, char *argv[], s /* host::path */ if (strncmp(cp, f->host, len) || (cp[len] != ':' && cp[len] != '\0')) - errx(1, "different remote host: %s", + errx(ERR_SYNTAX, "different remote host: %s", f->sources[i]); memmove(f->sources[i], f->sources[i] + len + 2, j - len - 1); @@ -257,7 +260,7 @@ fargs_parse(size_t argc, char *argv[], s /* host:path */ if (strncmp(cp, f->host, len) || (cp[len] != ':' && cp[len] != '\0')) - errx(1, "different remote host: %s", + errx(ERR_SYNTAX, "different remote host: %s", f->sources[i]); memmove(f->sources[i], f->sources[i] + len + 1, j - len); @@ -267,62 +270,92 @@ fargs_parse(size_t argc, char *argv[], s return f; } +static struct opts opts; + +#define OP_ADDRESS 1000 +#define OP_PORT 1001 +#define OP_RSYNCPATH 1002 +#define OP_TIMEOUT 1003 +#define OP_VERSION 1004 +#define OP_EXCLUDE 1005 +#define OP_INCLUDE 1006 +#define OP_EXCLUDE_FROM 1007 +#define OP_INCLUDE_FROM 1008 +#define OP_COMP_DEST 1009 +#define OP_COPY_DEST 1010 +#define OP_LINK_DEST 1011 +#define OP_MAX_SIZE 1012 +#define OP_MIN_SIZE 1013 + +const struct option lopts[] = { + { "address", required_argument, NULL, OP_ADDRESS }, + { "archive", no_argument, NULL, 'a' }, + { "compare-dest", required_argument, NULL, OP_COMP_DEST }, +#if 0 + { "copy-dest", required_argument, NULL, OP_COPY_DEST }, + { "link-dest", required_argument, NULL, OP_LINK_DEST }, +#endif + { "compress", no_argument, NULL, 'z' }, + { "del", no_argument, &opts.del, 1 }, + { "delete", no_argument, &opts.del, 1 }, + { "devices", no_argument, &opts.devices, 1 }, + { "no-devices", no_argument, &opts.devices, 0 }, + { "dry-run", no_argument, &opts.dry_run, 1 }, + { "exclude", required_argument, NULL, OP_EXCLUDE }, + { "exclude-from", required_argument, NULL, OP_EXCLUDE_FROM }, + { "group", no_argument, &opts.preserve_gids, 1 }, + { "no-group", no_argument, &opts.preserve_gids, 0 }, + { "help", no_argument, NULL, 'h' }, + { "include", required_argument, NULL, OP_INCLUDE }, + { "include-from", required_argument, NULL, OP_INCLUDE_FROM }, + { "links", no_argument, &opts.preserve_links, 1 }, + { "max-size", required_argument, NULL, OP_MAX_SIZE }, + { "min-size", required_argument, NULL, OP_MIN_SIZE }, + { "no-links", no_argument, &opts.preserve_links, 0 }, + { "no-motd", no_argument, &opts.no_motd, 1 }, + { "numeric-ids", no_argument, &opts.numeric_ids, 1 }, + { "owner", no_argument, &opts.preserve_uids, 1 }, + { "no-owner", no_argument, &opts.preserve_uids, 0 }, + { "perms", no_argument, &opts.preserve_perms, 1 }, + { "no-perms", no_argument, &opts.preserve_perms, 0 }, + { "port", required_argument, NULL, OP_PORT }, + { "recursive", no_argument, &opts.recursive, 1 }, + { "no-recursive", no_argument, &opts.recursive, 0 }, + { "rsh", required_argument, NULL, 'e' }, + { "rsync-path", required_argument, NULL, OP_RSYNCPATH }, + { "sender", no_argument, &opts.sender, 1 }, + { "server", no_argument, &opts.server, 1 }, + { "specials", no_argument, &opts.specials, 1 }, + { "no-specials", no_argument, &opts.specials, 0 }, + { "timeout", required_argument, NULL, OP_TIMEOUT }, + { "times", no_argument, &opts.preserve_times, 1 }, + { "no-times", no_argument, &opts.preserve_times, 0 }, + { "verbose", no_argument, &verbose, 1 }, + { "no-verbose", no_argument, &verbose, 0 }, + { "version", no_argument, NULL, OP_VERSION }, + { NULL, 0, NULL, 0 } +}; + int main(int argc, char *argv[]) { - struct opts opts; pid_t child; - int fds[2], sd = -1, rc, c, st, i; - struct sess sess; + int fds[2], sd = -1, rc, c, st, i, lidx; + size_t basedir_cnt = 0; + struct sess sess; struct fargs *fargs; char **args; - const char *errstr; - const struct option lopts[] = { - { "port", required_argument, NULL, 3 }, - { "rsh", required_argument, NULL, 'e' }, - { "rsync-path", required_argument, NULL, 1 }, - { "sender", no_argument, &opts.sender, 1 }, - { "server", no_argument, &opts.server, 1 }, - { "dry-run", no_argument, &opts.dry_run, 1 }, - { "version", no_argument, NULL, 2 }, - { "archive", no_argument, NULL, 'a' }, - { "help", no_argument, NULL, 'h' }, - { "compress", no_argument, NULL, 'z' }, - { "del", no_argument, &opts.del, 1 }, - { "delete", no_argument, &opts.del, 1 }, - { "devices", no_argument, &opts.devices, 1 }, - { "no-devices", no_argument, &opts.devices, 0 }, - { "group", no_argument, &opts.preserve_gids, 1 }, - { "no-group", no_argument, &opts.preserve_gids, 0 }, - { "links", no_argument, &opts.preserve_links, 1 }, - { "no-links", no_argument, &opts.preserve_links, 0 }, - { "owner", no_argument, &opts.preserve_uids, 1 }, - { "no-owner", no_argument, &opts.preserve_uids, 0 }, - { "perms", no_argument, &opts.preserve_perms, 1 }, - { "no-perms", no_argument, &opts.preserve_perms, 0 }, - { "numeric-ids", no_argument, &opts.numeric_ids, 1 }, - { "recursive", no_argument, &opts.recursive, 1 }, - { "no-recursive", no_argument, &opts.recursive, 0 }, - { "specials", no_argument, &opts.specials, 1 }, - { "no-specials", no_argument, &opts.specials, 0 }, - { "timeout", required_argument, NULL, 5 }, - { "times", no_argument, &opts.preserve_times, 1 }, - { "no-times", no_argument, &opts.preserve_times, 0 }, - { "verbose", no_argument, &verbose, 1 }, - { "no-verbose", no_argument, &verbose, 0 }, - { "address", required_argument, NULL, 4 }, - { "no-motd", no_argument, NULL, 6 }, - { NULL, 0, NULL, 0 }}; + const char *errstr; /* Global pledge. */ if (pledge("stdio unix rpath wpath cpath dpath inet fattr chown dns getpw proc exec unveil", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); - memset(&opts, 0, sizeof(struct opts)); + opts.max_size = opts.min_size = -1; - while ((c = getopt_long(argc, argv, "Dae:ghlnoprtvxz", lopts, NULL)) + while ((c = getopt_long(argc, argv, "Dae:ghlnoprtvxz", lopts, &lidx)) != -1) { switch (c) { case 'D': @@ -375,27 +408,84 @@ main(int argc, char *argv[]) case 0: /* Non-NULL flag values (e.g., --sender). */ break; - case 1: - opts.rsync_path = optarg; + case OP_ADDRESS: + opts.address = optarg; break; - case 2: - fprintf(stderr, "openrsync: protocol version %u\n", - RSYNC_PROTOCOL); - exit(0); - case 3: + case OP_PORT: opts.port = optarg; break; - case 4: - opts.address = optarg; + case OP_RSYNCPATH: + opts.rsync_path = optarg; break; - case 5: + case OP_TIMEOUT: poll_timeout = strtonum(optarg, 0, 60*60, &errstr); if (errstr != NULL) - errx(1, "timeout is %s: %s", errstr, optarg); + errx(ERR_SYNTAX, "timeout is %s: %s", + errstr, optarg); break; - case 6: - opts.no_motd = 1; + case OP_EXCLUDE: + if (parse_rule(optarg, RULE_EXCLUDE) == -1) + errx(ERR_SYNTAX, "syntax error in exclude: %s", + optarg); + break; + case OP_INCLUDE: + if (parse_rule(optarg, RULE_INCLUDE) == -1) + errx(ERR_SYNTAX, "syntax error in include: %s", + optarg); + break; + case OP_EXCLUDE_FROM: + parse_file(optarg, RULE_EXCLUDE); + break; + case OP_INCLUDE_FROM: + parse_file(optarg, RULE_INCLUDE); + break; + case OP_COMP_DEST: + if (opts.alt_base_mode !=0 && + opts.alt_base_mode != BASE_MODE_COMPARE) { + errx(1, "option --%s conflicts with %s", + lopts[lidx].name, + alt_base_mode(opts.alt_base_mode)); + } + opts.alt_base_mode = BASE_MODE_COMPARE; +#if 0 + goto basedir; + case OP_COPY_DEST: + if (opts.alt_base_mode !=0 && + opts.alt_base_mode != BASE_MODE_COPY) { + errx(1, "option --%s conflicts with %s", + lopts[lidx].name, + alt_base_mode(opts.alt_base_mode)); + } + opts.alt_base_mode = BASE_MODE_COPY; + goto basedir; + case OP_LINK_DEST: + if (opts.alt_base_mode !=0 && + opts.alt_base_mode != BASE_MODE_LINK) { + errx(1, "option --%s conflicts with %s", + lopts[lidx].name, + alt_base_mode(opts.alt_base_mode)); + } + opts.alt_base_mode = BASE_MODE_LINK; + +basedir: +#endif + if (basedir_cnt >= MAX_BASEDIR) + errx(1, "too many --%s directories specified", + lopts[lidx].name); + opts.basedir[basedir_cnt++] = optarg; + break; + case OP_MAX_SIZE: + if (scan_scaled(optarg, &opts.max_size) == -1) + err(1, "bad max-size"); + break; + case OP_MIN_SIZE: + if (scan_scaled(optarg, &opts.min_size) == -1) + err(1, "bad min-size"); break; + case OP_VERSION: + fprintf(stderr, "openrsync: protocol version %u\n", + RSYNC_PROTOCOL); + exit(0); case 'h': default: goto usage; @@ -415,8 +505,7 @@ main(int argc, char *argv[]) /* by default and for --timeout=0 disable poll_timeout */ if (poll_timeout == 0) - poll_timeout = -1; - else + poll_timeout = -1; else poll_timeout *= 1000; /* @@ -461,43 +550,36 @@ main(int argc, char *argv[]) if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw proc exec unveil", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); /* Create a bidirectional socket and start our child. */ if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0, fds) == -1) - err(1, "socketpair"); + err(ERR_IPC, "socketpair"); switch ((child = fork())) { case -1: - err(1, "fork"); + err(ERR_IPC, "fork"); case 0: close(fds[0]); if (pledge("stdio exec", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); memset(&sess, 0, sizeof(struct sess)); sess.opts = &opts; - if ((args = fargs_cmdline(&sess, fargs, NULL)) == NULL) { - ERRX1("fargs_cmdline"); - _exit(1); - } + args = fargs_cmdline(&sess, fargs, NULL); for (i = 0; args[i] != NULL; i++) LOG2("exec[%d] = %s", i, args[i]); /* Make sure the child's stdin is from the sender. */ - if (dup2(fds[1], STDIN_FILENO) == -1) { - ERR("dup2"); - _exit(1); - } - if (dup2(fds[1], STDOUT_FILENO) == -1) { - ERR("dup2"); - _exit(1); - } + if (dup2(fds[1], STDIN_FILENO) == -1) + err(ERR_IPC, "dup2"); + if (dup2(fds[1], STDOUT_FILENO) == -1) + err(ERR_IPC, "dup2"); execvp(args[0], args); - _exit(1); + _exit(ERR_IPC); /* NOTREACHED */ default: close(fds[1]); @@ -511,7 +593,7 @@ main(int argc, char *argv[]) close(fds[0]); if (waitpid(child, &st, 0) == -1) - err(1, "waitpid"); + err(ERR_WAITPID, "waitpid"); /* * If we don't already have an error (rc == 0), then inherit the @@ -519,18 +601,23 @@ main(int argc, char *argv[]) * If it hasn't exited, it overrides our return value. */ - if (WIFEXITED(st) && rc == 0) - rc = WEXITSTATUS(st); - else if (!WIFEXITED(st)) - rc = 1; + if (rc == 0) { + if (WIFEXITED(st)) + rc = WEXITSTATUS(st); + else if (WIFSIGNALED(st)) + rc = ERR_TERMIMATED; + else + rc = ERR_WAITPID; + } exit(rc); usage: fprintf(stderr, "usage: %s" - " [-aDglnoprtvx] [-e program] [--address=sourceaddr] [--del]\n" - "\t[--no-motd] [--numeric-ids] [--port=portnumber] " - "[--rsync-path=program]\n\t[--timeout=seconds] [--version] " - "source ... directory\n", + " [-aDglnoprtvx] [-e program] [--address=sourceaddr]\n" + "\t[--compare-dest=dir] [--del] [--exclude] [--exclude-from=file]\n" + "\t[--include] [--include-from=file] [--no-motd] [--numeric-ids]\n" + "\t[--port=portnumber] [--rsync-path=program] [--timeout=seconds]\n" + "\t[--version] source ... directory\n", getprogname()); - exit(1); + exit(ERR_SYNTAX); } Index: usr.bin/rsync/misc.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/misc.c,v retrieving revision 1.2 diff -u -p -u -r1.2 misc.c --- usr.bin/rsync/misc.c 22 Mar 2021 11:14:42 -0000 1.2 +++ usr.bin/rsync/misc.c 6 Nov 2021 18:30:35 -0000 @@ -45,7 +45,7 @@ addargs(arglist *args, const char *fmt, r = vasprintf(&cp, fmt, ap); va_end(ap); if (r == -1) - err(1, "addargs: argument too long"); + err(ERR_NOMEM, "addargs: argument too long"); nalloc = args->nalloc; if (args->list == NULL) { @@ -54,9 +54,10 @@ addargs(arglist *args, const char *fmt, } else if (args->num+2 >= nalloc) nalloc *= 2; - args->list = recallocarray(args->list, args->nalloc, nalloc, sizeof(char *)); + args->list = recallocarray(args->list, args->nalloc, nalloc, + sizeof(char *)); if (!args->list) - err(1, "malloc"); + err(ERR_NOMEM, NULL); args->nalloc = nalloc; args->list[args->num++] = cp; args->list[args->num] = NULL; Index: usr.bin/rsync/mkpath.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/mkpath.c,v retrieving revision 1.4 diff -u -p -u -r1.4 mkpath.c --- usr.bin/rsync/mkpath.c 8 May 2019 21:30:11 -0000 1.4 +++ usr.bin/rsync/mkpath.c 6 Nov 2021 18:31:07 -0000 @@ -46,28 +46,34 @@ mkpath(char *path) { struct stat sb; char *slash; - int done = 0; + int done; slash = path; - while (!done) { + for (;;) { slash += strspn(slash, "/"); slash += strcspn(slash, "/"); done = (*slash == '\0'); *slash = '\0'; - if (stat(path, &sb)) { - if (errno != ENOENT || (mkdir(path, 0777) && - errno != EEXIST)) { - ERR("%s: stat", path); + if (mkdir(path, 0777) != 0) { + int mkdir_errno = errno; + + if (stat(path, &sb) == -1) { + /* Not there; use mkdir()s errno */ + errno = mkdir_errno; + return (-1); + } + if (!S_ISDIR(sb.st_mode)) { + /* Is there, but isn't a directory */ + errno = ENOTDIR; return (-1); } - } else if (!S_ISDIR(sb.st_mode)) { - errno = ENOTDIR; - ERR("%s: stat", path); - return (-1); } + + if (done) + break; *slash = '/'; } Index: usr.bin/rsync/receiver.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/receiver.c,v retrieving revision 1.25 diff -u -p -u -r1.25 receiver.c --- usr.bin/rsync/receiver.c 24 Nov 2020 16:54:44 -0000 1.25 +++ usr.bin/rsync/receiver.c 6 Nov 2021 18:35:51 -0000 @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -171,7 +172,7 @@ int rsync_receiver(struct sess *sess, int fdin, int fdout, const char *root) { struct flist *fl = NULL, *dfl = NULL; - size_t i, flsz = 0, dflsz = 0, excl; + size_t i, flsz = 0, dflsz = 0; char *tofree; int rc = 0, dfd = -1, phase = 0, c; int32_t ioerror; @@ -180,29 +181,55 @@ rsync_receiver(struct sess *sess, int fd struct upload *ul = NULL; mode_t oumask; - if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw unveil", NULL) == -1) { - ERR("pledge"); - goto out; - } + if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw unveil", NULL) == -1) + err(ERR_IPC, "pledge"); - /* Client sends zero-length exclusions. */ + /* + * Create the path for our destination directory, if we're not + * in dry-run mode (which would otherwise crash w/the pledge). + * This uses our current umask: we might set the permissions on + * this directory in post_dir(). + */ - if (!sess->opts->server && - !io_write_int(sess, fdout, 0)) { - ERRX1("io_write_int"); - goto out; + if (!sess->opts->dry_run) { + if ((tofree = strdup(root)) == NULL) + err(ERR_NOMEM, NULL); + if (mkpath(tofree) < 0) + err(ERR_FILE_IO, "%s: mkpath", tofree); + free(tofree); } - if (sess->opts->server && sess->opts->del) { - if (!io_read_size(sess, fdin, &excl)) { - ERRX1("io_read_size"); - goto out; - } else if (excl != 0) { - ERRX("exclusion list is non-empty"); - goto out; - } + /* + * Make our entire view of the file-system be limited to what's + * in the root directory. + * This prevents us from accidentally (or "under the influence") + * writing into other parts of the file-system. + */ + if (sess->opts->basedir[0]) { + /* + * XXX just unveil everything for read + * Could unveil each basedir or maybe a common path + * also the fact that relative path are relative to the + * root does not help. + */ + if (unveil("/", "r") == -1) + err(ERR_IPC, "%s: unveil", root); } + if (unveil(root, "rwc") == -1) + err(ERR_IPC, "%s: unveil", root); + + if (unveil(NULL, NULL) == -1) + err(ERR_IPC, "unveil"); + + /* Client sends exclusions. */ + if (!sess->opts->server) + send_rules(sess, fdout); + + /* Server receives exclusions if delete is on. */ + if (sess->opts->server && sess->opts->del) + recv_rules(sess, fdin); + /* * Start by receiving the file list and our mystery number. * These we're going to be touching on our local system. @@ -233,25 +260,6 @@ rsync_receiver(struct sess *sess, int fd LOG2("%s: receiver destination", root); /* - * Create the path for our destination directory, if we're not - * in dry-run mode (which would otherwise crash w/the pledge). - * This uses our current umask: we might set the permissions on - * this directory in post_dir(). - */ - - if (!sess->opts->dry_run) { - if ((tofree = strdup(root)) == NULL) { - ERR("strdup"); - goto out; - } else if (mkpath(tofree) < 0) { - ERRX1("%s: mkpath", root); - free(tofree); - goto out; - } - free(tofree); - } - - /* * Disable umask() so we can set permissions fully. * Then open the directory iff we're not in dry_run. */ @@ -259,11 +267,9 @@ rsync_receiver(struct sess *sess, int fd oumask = umask(0); if (!sess->opts->dry_run) { - dfd = open(root, O_RDONLY | O_DIRECTORY, 0); - if (dfd == -1) { - ERR("%s: open", root); - goto out; - } + dfd = open(root, O_RDONLY | O_DIRECTORY); + if (dfd == -1) + err(ERR_FILE_IO, "%s: open", root); } /* @@ -278,21 +284,6 @@ rsync_receiver(struct sess *sess, int fd goto out; } - /* - * Make our entire view of the file-system be limited to what's - * in the root directory. - * This prevents us from accidentally (or "under the influence") - * writing into other parts of the file-system. - */ - - if (unveil(root, "rwc") == -1) { - ERR("%s: unveil", root); - goto out; - } else if (unveil(NULL, NULL) == -1) { - ERR("%s: unveil", root); - goto out; - } - /* If we have a local set, go for the deletion. */ if (!flist_del(sess, dfd, dfl, dflsz)) { @@ -356,7 +347,7 @@ rsync_receiver(struct sess *sess, int fd */ if (sess->mplex_reads && - (POLLIN & pfd[PFD_SENDER_IN].revents)) { + (pfd[PFD_SENDER_IN].revents & POLLIN)) { if (!io_read_flush(sess, fdin)) { ERRX1("io_read_flush"); goto out; @@ -371,8 +362,8 @@ rsync_receiver(struct sess *sess, int fd * is read to mmap. */ - if ((POLLIN & pfd[PFD_UPLOADER_IN].revents) || - (POLLOUT & pfd[PFD_SENDER_OUT].revents)) { + if ((pfd[PFD_UPLOADER_IN].revents & POLLIN) || + (pfd[PFD_SENDER_OUT].revents & POLLOUT)) { c = rsync_uploader(ul, &pfd[PFD_UPLOADER_IN].fd, sess, &pfd[PFD_SENDER_OUT].fd); @@ -391,8 +382,8 @@ rsync_receiver(struct sess *sess, int fd * messages, which will otherwise clog up the pipes. */ - if ((POLLIN & pfd[PFD_SENDER_IN].revents) || - (POLLIN & pfd[PFD_DOWNLOADER_IN].revents)) { + if ((pfd[PFD_SENDER_IN].revents & POLLIN) || + (pfd[PFD_DOWNLOADER_IN].revents & POLLIN)) { c = rsync_downloader(dl, sess, &pfd[PFD_DOWNLOADER_IN].fd); if (c < 0) { @@ -421,10 +412,12 @@ rsync_receiver(struct sess *sess, int fd if (!io_write_int(sess, fdout, -1)) { ERRX1("io_write_int"); goto out; - } else if (!io_read_int(sess, fdin, &ioerror)) { + } + if (!io_read_int(sess, fdin, &ioerror)) { ERRX1("io_read_int"); goto out; - } else if (ioerror != -1) { + } + if (ioerror != -1) { ERRX("expected phase ack"); goto out; } @@ -445,7 +438,8 @@ rsync_receiver(struct sess *sess, int fd if (!sess_stats_recv(sess, fdin)) { ERRX1("sess_stats_recv"); goto out; - } else if (!io_write_int(sess, fdout, -1)) { + } + if (!io_write_int(sess, fdout, -1)) { ERRX1("io_write_int"); goto out; } Index: usr.bin/rsync/rmatch.c =================================================================== RCS file: usr.bin/rsync/rmatch.c diff -N usr.bin/rsync/rmatch.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ usr.bin/rsync/rmatch.c 6 Nov 2021 14:21:39 -0000 @@ -0,0 +1,396 @@ +/* $OpenBSD: rmatch.c,v 1.2 2021/08/29 15:37:58 claudio Exp $ */ + +/* + * Copyright (c) 2021 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * Copyright (c) 1989, 1993, 1994 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Guido van Rossum. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "charclass.h" +#include "extern.h" + +#define RANGE_MATCH 1 +#define RANGE_NOMATCH 0 +#define RANGE_ERROR (-1) + +static int +classmatch(const char *pattern, char test, const char **ep) +{ + const char *mismatch = pattern; + const struct cclass *cc; + const char *colon; + size_t len; + int rval = RANGE_NOMATCH; + + if (*pattern++ != ':') { + *ep = mismatch; + return RANGE_ERROR; + } + if ((colon = strchr(pattern, ':')) == NULL || colon[1] != ']') { + *ep = mismatch; + return RANGE_ERROR; + } + *ep = colon + 2; + len = (size_t)(colon - pattern); + + for (cc = cclasses; cc->name != NULL; cc++) { + if (!strncmp(pattern, cc->name, len) && cc->name[len] == '\0') { + if (cc->isctype((unsigned char)test)) + rval = RANGE_MATCH; + return rval; + } + } + + /* invalid character class, treat as normal text */ + *ep = mismatch; + return RANGE_ERROR; +} + +static int +rangematch(const char **pp, char test) +{ + const char *pattern = *pp; + int negate, ok; + char c, c2; + + /* + * A bracket expression starting with an unquoted circumflex + * character produces unspecified results (IEEE 1003.2-1992, + * 3.13.2). This implementation treats it like '!', for + * consistency with the regular expression syntax. + * J.T. Conklin (conklin@ngai.kaleida.com) + */ + if ((negate = (*pattern == '!' || *pattern == '^'))) + ++pattern; + + /* + * A right bracket shall lose its special meaning and represent + * itself in a bracket expression if it occurs first in the list. + * -- POSIX.2 2.8.3.2 + */ + ok = 0; + c = *pattern++; + do { + if (c == '[') { + switch (classmatch(pattern, test, &pattern)) { + case RANGE_MATCH: + ok = 1; + continue; + case RANGE_NOMATCH: + continue; + default: + /* invalid character class, treat litterally. */ + break; + } + } + if (c == '\\') + c = *pattern++; + if (c == '\0') + return RANGE_ERROR; + /* patterns can not match on '/' */ + if (c == '/') + return RANGE_NOMATCH; + if (*pattern == '-' + && (c2 = *(pattern + 1)) != '\0' && c2 != ']') { + pattern += 2; + if (c2 == '\\') + c2 = *pattern++; + if (c2 == '\0') + return RANGE_ERROR; + if (c <= test && test <= c2) + ok = 1; + } else if (c == test) + ok = 1; + } while ((c = *pattern++) != ']'); + + *pp = pattern; + return (ok == negate ? RANGE_NOMATCH : RANGE_MATCH); +} + +/* + * Single character match, advances pattern as much as needed. + * Return 0 on match and !0 (aka 1) on missmatch. + * When matched pp is advanced to the end of the pattern matched. + */ +static int +matchchar(const char **pp, const char in) +{ + const char *pattern = *pp; + char c; + int rv = 0; + + switch (c = *pattern++) { + case '?': + if (in == '\0') + rv = 1; + if (in == '/') + rv = 1; + break; + case '[': + if (in == '\0') + rv = 1; + if (in == '/') + rv = 1; + if (rv == 1) + break; + + switch (rangematch(&pattern, in)) { + case RANGE_ERROR: + /* not a good range, treat as normal text */ + goto normal; + case RANGE_MATCH: + break; + case RANGE_NOMATCH: + rv = 1; + } + break; + case '\\': + if ((c = *pattern++) == '\0') { + c = '\\'; + --pattern; + } + /* FALLTHROUGH */ + default: + normal: + if (c != in) + rv = 1; + break; + } + + *pp = pattern; + return rv; +} + +/* + * Do a substring match. If wild is set then the pattern started with a '*'. + * The match will go until '*', '/' or '\0' is encountered in pattern or + * the input string is consumed up to end. + * The pattern and string handles pp and ss are updated only on success. + */ +static int +matchsub(const char **pp, const char **ss, const char *end, int wild) +{ + const char *pattern = *pp; + const char *p = pattern; + const char *string = *ss; + size_t matchlen; + + /* first calculate how many characters the submatch will consume */ + for (matchlen = 0; *p != '\0'; matchlen++) { + if (p[0] == '*') + break; + /* '/' acts as barrier */ + if (p[0] == '/' || (p[0] == '\\' && p[1] == '/')) { + if (wild) { + /* match needs to match up to end of segment */ + if (string > end - matchlen) + return 1; + string = end - matchlen; + wild = 0; + } + break; + } + /* + * skip forward one character in pattern by doing a + * dummy lookup. + */ + matchchar(&p, ' '); + } + + /* not enough char to match */ + if (string > end - matchlen) + return 1; + + if (*p == '\0') { + if (wild) { + /* match needs to match up to end of segment */ + string = end - matchlen; + wild = 0; + } + } + + while (*pattern != '\0' && *pattern != '*') { + /* eat possible escape char before '/' */ + if (pattern[0] == '\\' && pattern[1] == '/') + pattern++; + if (pattern[0] == '/') + break; + + /* check if there are still characters available to compare */ + if (string >= end) + return 1; + /* Compare one char at a time. */ + if (!matchchar(&pattern, *string++)) + continue; + if (wild) { + /* skip forward one char and restart match */ + string = ++*ss; + pattern = *pp; + /* can it still match? */ + if (string > end - matchlen) + return 1; + } else { + /* failed match */ + return 1; + } + } + + *pp = pattern; + *ss = string; + return 0; +} + +/* + * File matching with the addition of the special '**'. + * Returns 0 on match and !0 for strings that do not match pattern. + */ +int +rmatch(const char *pattern, const char *string, int leading_dir) +{ + const char *segend, *segnext, *mismatch = NULL; + int wild, starstar; + + while (*pattern && *string) { + + /* handle leading '/' first */ + if (pattern[0] == '\\' && pattern[1] == '/') + pattern++; + if (*string == '/' && *pattern == '/') { + string++; + pattern++; + } + + /* match to the next '/' in string */ + segend = strchr(string, '/'); + if (segend == NULL) + segend = strchr(string, '\0'); + + while (*pattern) { + /* + * Check for '*' and '**'. For '*' reduce '*' and '?' + * sequences into n-'?' and trailing '*'. + * For '**' this optimisation can not be done + * since '**???/' will match 'a/aa/aaa/' but not + * 'a/aa/aa/' still additional '*' will be reduced. + */ + wild = 0; + starstar = 0; + for ( ; *pattern == '*' || *pattern == '?'; pattern++) { + if (pattern[0] == '*') { + if (pattern[1] == '*') { + starstar = 1; + pattern++; + } + wild = 1; + } else if (!starstar) { /* pattern[0] == '?' */ + if (string < segend && *string != '/') + string++; + else + /* no match possible */ + return 1; + } else + break; + } + + /* pattern ends in '**' so it is a match */ + if (starstar && *pattern == '\0') + return 0; + + if (starstar) { + segnext = segend; + mismatch = pattern; + } + + while (string < segend) { + if (matchsub(&pattern, &string, segend, wild)) { +failed_match: + /* + * failed to match, if starstar retry + * with the next segment. + */ + if (mismatch) { + pattern = mismatch; + wild = 1; + string = segnext; + if (*string == '/') + string++; + segend = strchr(string, '/'); + if (!segend) + segend = strchr(string, + '\0'); + segnext = segend; + if (string < segend) + continue; + } + /* no match possible */ + return 1; + } + break; + } + + /* at end of string segment, eat up any extra '*' */ + if (string >= segend && *pattern != '*') + break; + } + if (*string != '\0' && *string != '/') + goto failed_match; + if (*pattern != '\0' && *pattern != '/') + goto failed_match; + } + + /* if both pattern and string are consumed it was a match */ + if (*pattern == '\0' && *string == '\0') + return 0; + /* if leading_dir is set then string can also be '/' for success */ + if (leading_dir && *pattern == '\0' && *string == '/') + return 0; + /* else failure */ + return 1; +} Index: usr.bin/rsync/rsync.1 =================================================================== RCS file: /cvs/src/usr.bin/rsync/rsync.1,v retrieving revision 1.24 diff -u -p -u -r1.24 rsync.1 --- usr.bin/rsync/rsync.1 31 Mar 2021 20:36:05 -0000 1.24 +++ usr.bin/rsync/rsync.1 6 Nov 2021 18:30:54 -0000 @@ -14,7 +14,7 @@ .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. .\" -.Dd $Mdocdate: March 31 2021 $ +.Dd $Mdocdate: October 29 2021 $ .Dt OPENRSYNC 1 .Os .Sh NAME @@ -25,7 +25,14 @@ .Op Fl aDglnoprtvx .Op Fl e Ar program .Op Fl -address Ns = Ns Ar sourceaddr +.Op Fl -compare-dest Ns = Ns Ar directory .Op Fl -del +.Op Fl -exclude Ar pattern +.Op Fl -exclude-from Ns = Ns Ar file +.Op Fl -include Ar pattern +.Op Fl -include-from Ns = Ns Ar file +.Op Fl -max-size Ns = Ns size +.Op Fl -min-size Ns = Ns size .Op Fl -no-motd .Op Fl -numeric-ids .Op Fl -port Ns = Ns Ar service @@ -58,6 +65,18 @@ When connecting to an rsync daemon, use .Ar sourceaddr as the source address for connections, which is useful on machines with multiple interfaces. +.It Fl -compare-dest Ns = Ns Ar directory +Use directory as an alternate base directory to compare files against on the +destination machine. +If file in +.Ar directory +is found and identical to the sender's file, the file will not be transferred. +Multiple +.Fl -compare-dest +directories may be provided. +If +.Ar directory +is a relative path, it is relative to the destination directory. .It Fl D Also transfer device and special files. Shorthand for @@ -70,6 +89,26 @@ not found in directories. Only applicable with .Fl r . +.It Fl -exclude Ar pattern +Exclude files matching +.Em pattern . +.It Fl -exclude-from Ns = Ns Ar file +Load +.Em patterns +and +.Em rules +from +.Em file . +.It Fl -include Ar pattern +Include files matching +.Em pattern . +.It Fl -include-from Ns = Ns Ar file +Load +.Em patterns +and +.Em rules +from +.Em file . .It Fl -devices Also transfer device files. .It Fl e Ar program , Fl -rsh Ns = Ns Ar program @@ -90,6 +129,22 @@ set the numeric group ID to match the so Also transfer symbolic links. The link is transferred as a standalone file: if the destination does not exist, it will be broken. +.It Fl -max-size Ar size +Don't transfer any file that is larger than +.Ar size +bytes. +Alternatively +.Ar size +may instead use a multiplier, as documented in +.Xr scan_scaled 3 , +to specify the size. +.It Fl -min-size Ar size +Don't transfer any file that is smaller than +.Ar size +bytes. +See +.Fl -max-size +on the definiton of size. .It Fl n , -dry-run Do not actually modify the destination. Mainly useful in combination with @@ -210,6 +265,7 @@ or symbolic links The destination .Ar directory must be a directory and is created if not found. +.\" .Sh PATTERNS AND RULES .\" .Sh ENVIRONMENT .\" .Sh FILES .Sh EXIT STATUS Index: usr.bin/rsync/rules.c =================================================================== RCS file: usr.bin/rsync/rules.c diff -N usr.bin/rsync/rules.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ usr.bin/rsync/rules.c 6 Nov 2021 14:21:39 -0000 @@ -0,0 +1,495 @@ +/* $OpenBSD: rules.c,v 1.4 2021/11/03 14:42:12 deraadt Exp $ */ +/* + * Copyright (c) 2021 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include +#include +#include +#include + +#include "extern.h" + +struct rule { + char *pattern; + enum rule_type type; +#ifdef NOTYET + unsigned int modifiers; +#endif + short numseg; + unsigned char anchored; + unsigned char fileonly; + unsigned char nowild; + unsigned char onlydir; + unsigned char leadingdir; +}; + +static struct rule *rules; +static size_t numrules; /* number of rules */ +static size_t rulesz; /* available size */ + +/* up to protocol 29 filter rules only support - + ! and no modifiers */ + +const struct command { + enum rule_type type; + char sopt; + const char *lopt; +} commands[] = { + { RULE_EXCLUDE, '-', "exclude" }, + { RULE_INCLUDE, '+', "include" }, + { RULE_CLEAR, '!', "clear" }, +#ifdef NOTYET + { RULE_MERGE, '.', "merge" }, + { RULE_DIR_MERGE, ':', "dir-merge" }, + { RULE_SHOW, 'S', "show" }, + { RULE_HIDE, 'H', "hide" }, + { RULE_PROTECT, 'P', "protect" }, + { RULE_RISK, 'R', "risk" }, +#endif + { 0 } +}; + +#ifdef NOTYET +#define MOD_ABSOLUTE 0x0001 +#define MOD_NEGATE 0x0002 +#define MOD_CVSEXCLUDE 0x0004 +#define MOD_SENDING 0x0008 +#define MOD_RECEIVING 0x0010 +#define MOD_PERISHABLE 0x0020 +#define MOD_XATTR 0x0040 +#define MOD_MERGE_EXCLUDE 0x0080 +#define MOD_MERGE_INCLUDE 0x0100 +#define MOD_MERGE_CVSCOMPAT 0x0200 +#define MOD_MERGE_EXCLUDE_FILE 0x0400 +#define MOD_MERGE_NO_INHERIT 0x0800 +#define MOD_MERGE_WORDSPLIT 0x1000 + +/* maybe support absolute and negate */ +const struct modifier { + unsigned int modifier; + char sopt; +} modifiers[] = { + { MOD_ABSOLUTE, '/' }, + { MOD_NEGATE, '!' }, + { MOD_CVSEXCLUDE, 'C' }, + { MOD_SENDING, 's' }, + { MOD_RECEIVING, 'r' }, + { MOD_PERISHABLE, 'p' }, + { MOD_XATTR, 'x' }, + /* for '.' and ':' types */ + { MOD_MERGE_EXCLUDE, '-' }, + { MOD_MERGE_INCLUDE, '+' }, + { MOD_MERGE_CVSCOMPAT, 'C' }, + { MOD_MERGE_EXCLUDE_FILE, 'e' }, + { MOD_MERGE_NO_INHERIT, 'n' }, + { MOD_MERGE_WORDSPLIT, 'w' }, + { 0 } +} +#endif + +static struct rule * +get_next_rule(void) +{ + struct rule *new; + size_t newsz; + + if (++numrules > rulesz) { + if (rulesz == 0) + newsz = 16; + else + newsz = rulesz * 2; + + new = recallocarray(rules, rulesz, newsz, sizeof(*rules)); + if (new == NULL) + err(ERR_NOMEM, NULL); + + rules = new; + rulesz = newsz; + } + + return rules + numrules - 1; +} + +static enum rule_type +parse_command(const char *command, size_t len) +{ + const char *mod; + size_t i; + + mod = memchr(command, ',', len); + if (mod != NULL) { + /* XXX modifiers not yet implemented */ + return RULE_NONE; + } + + for (i = 0; commands[i].type != RULE_NONE; i++) { + if (strncmp(commands[i].lopt, command, len) == 0) + return commands[i].type; + if (len == 1 && commands[i].sopt == *command) + return commands[i].type; + } + + return RULE_NONE; +} + +static void +parse_pattern(struct rule *r, char *pattern) +{ + size_t plen; + char *p; + short nseg = 1; + + /* + * check for / at start and end of pattern both are special and + * can bypass full path matching. + */ + if (*pattern == '/') { + pattern++; + r->anchored = 1; + } + plen = strlen(pattern); + /* + * check for patterns ending in '/' and '/'+'***' and handle them + * specially. Because of this and the check above pattern will never + * start or end with a '/'. + */ + if (plen > 1 && pattern[plen - 1] == '/') { + r->onlydir = 1; + pattern[plen - 1] = '\0'; + } + if (plen > 4 && strcmp(pattern + plen - 4, "/***") == 0) { + r->leadingdir = 1; + pattern[plen - 4] = '\0'; + } + + /* count how many segments the pattern has. */ + for (p = pattern; *p != '\0'; p++) + if (*p == '/') + nseg++; + r->numseg = nseg; + + /* check if this pattern only matches against the basename */ + if (nseg == 1 && !r->anchored) + r->fileonly = 1; + + if (strpbrk(pattern, "*?[") == NULL) { + /* no wildchar matching */ + r->nowild = 1; + } else { + /* requires wildchar matching */ + if (strstr(pattern, "**") != NULL) + r->numseg = -1; + } + + r->pattern = strdup(pattern); + if (r->pattern == NULL) + err(ERR_NOMEM, NULL); +} + +int +parse_rule(char *line, enum rule_type def) +{ + enum rule_type type; + struct rule *r; + char *pattern; + size_t len; + + switch (*line) { + case '#': + case ';': + /* comment */ + return 0; + case '\0': + /* ingore empty lines */ + return 0; + default: + len = strcspn(line, " _"); + type = parse_command(line, len); + if (type == RULE_NONE) { + if (def == RULE_NONE) + return -1; + type = def; + pattern = line; + } else + pattern = line + len + 1; + + if (*pattern == '\0' && type != RULE_CLEAR) + return -1; + if (*pattern != '\0' && type == RULE_CLEAR) + return -1; + break; + } + + r = get_next_rule(); + r->type = type; + parse_pattern(r, pattern); + + return 0; +} + +void +parse_file(const char *file, enum rule_type def) +{ + FILE *fp; + char *line = NULL; + size_t linesize = 0, linenum = 0; + ssize_t linelen; + + if ((fp = fopen(file, "r")) == NULL) + err(ERR_SYNTAX, "open: %s", file); + + while ((linelen = getline(&line, &linesize, fp)) != -1) { + linenum++; + line[linelen - 1] = '\0'; + if (parse_rule(line, def) == -1) + errx(ERR_SYNTAX, "syntax error in %s at entry %zu", + file, linenum); + } + + free(line); + if (ferror(fp)) + err(ERR_SYNTAX, "failed to parse file %s", file); + fclose(fp); +} + +static const char * +send_command(struct rule *r) +{ + static char buf[16]; + char *b = buf; + char *ep = buf + sizeof(buf); + + switch (r->type) { + case RULE_EXCLUDE: + *b++ = '-'; + break; + case RULE_INCLUDE: + *b++ = '+'; + break; + case RULE_CLEAR: + *b++ = '!'; + break; +#ifdef NOTYET + case RULE_MERGE: + *b++ = '.'; + break; + case RULE_DIR_MERGE: + *b++ = ':'; + break; + case RULE_SHOW: + *b++ = 'S'; + break; + case RULE_HIDE: + *b++ = 'H'; + break; + case RULE_PROTECT: + *b++ = 'P'; + break; + case RULE_RISK: + *b++ = 'R'; + break; +#endif + default: + err(ERR_SYNTAX, "unknown rule type %d", r->type); + } + +#ifdef NOTYET + for (i = 0; modifiers[i].modifier != 0; i++) { + if (rule->modifiers & modifiers[i].modifier) + *b++ = modifiers[i].sopt; + if (b >= ep - 3) + err(ERR_SYNTAX, "rule modifiers overflow"); + } +#endif + if (b >= ep - 3) + err(ERR_SYNTAX, "rule prefix overflow"); + *b++ = ' '; + + /* include the stripped root '/' for anchored patterns */ + if (r->anchored) + *b++ = '/'; + *b++ = '\0'; + return buf; +} + +static const char * +postfix_command(struct rule *r) +{ + static char buf[8]; + + buf[0] = '\0'; + if (r->onlydir) + strlcpy(buf, "/", sizeof(buf)); + if (r->leadingdir) + strlcpy(buf, "/***", sizeof(buf)); + + return buf; +} + +void +send_rules(struct sess *sess, int fd) +{ + const char *cmd; + const char *postfix; + struct rule *r; + size_t cmdlen, len, postlen, i; + + for (i = 0; i < numrules; i++) { + r = &rules[i]; + cmd = send_command(r); + if (cmd == NULL) + err(ERR_PROTOCOL, + "rules are incompatible with remote rsync"); + postfix = postfix_command(r); + cmdlen = strlen(cmd); + len = strlen(r->pattern); + postlen = strlen(postfix); + + if (!io_write_int(sess, fd, cmdlen + len + postlen)) + err(ERR_SOCK_IO, "send rules"); + if (!io_write_buf(sess, fd, cmd, cmdlen)) + err(ERR_SOCK_IO, "send rules"); + if (!io_write_buf(sess, fd, r->pattern, len)) + err(ERR_SOCK_IO, "send rules"); + /* include the '/' stripped by onlydir */ + if (postlen > 0) + if (!io_write_buf(sess, fd, postfix, postlen)) + err(ERR_SOCK_IO, "send rules"); + } + + if (!io_write_int(sess, fd, 0)) + err(ERR_SOCK_IO, "send rules"); +} + +void +recv_rules(struct sess *sess, int fd) +{ + char line[8192]; + size_t len; + + do { + if (!io_read_size(sess, fd, &len)) + err(ERR_SOCK_IO, "receive rules"); + + if (len == 0) + return; + if (len >= sizeof(line) - 1) + errx(ERR_SOCK_IO, "received rule too long"); + if (!io_read_buf(sess, fd, line, len)) + err(ERR_SOCK_IO, "receive rules"); + line[len] = '\0'; + if (parse_rule(line, RULE_NONE) == -1) + errx(ERR_PROTOCOL, "syntax error in received rules"); + } while (1); +} + +static inline int +rule_matched(struct rule *r) +{ + /* TODO apply negation once modifiers are added */ + + if (r->type == RULE_EXCLUDE) + return -1; + else + return 1; +} + +int +rules_match(const char *path, int isdir) +{ + const char *basename, *p = NULL; + struct rule *r; + size_t i; + + basename = strrchr(path, '/'); + if (basename != NULL) + basename += 1; + else + basename = path; + + for (i = 0; i < numrules; i++) { + r = &rules[i]; + + if (r->onlydir && !isdir) + continue; + + if (r->nowild) { + /* fileonly and anchored are mutually exclusive */ + if (r->fileonly) { + if (strcmp(basename, r->pattern) == 0) + return rule_matched(r); + } else if (r->anchored) { + /* + * assumes that neither path nor pattern + * start with a '/'. + */ + if (strcmp(path, r->pattern) == 0) + return rule_matched(r); + } else if (r->leadingdir) { + size_t plen = strlen(r->pattern); + + p = strstr(path, r->pattern); + /* + * match from start or dir boundary also + * match to end or to dir boundary + */ + if (p != NULL && (p == path || p[-1] == '/') && + (p[plen] == '\0' || p[plen] == '/')) + return rule_matched(r); + } else { + size_t len = strlen(path); + size_t plen = strlen(r->pattern); + + if (len >= plen && strcmp(path + len - plen, + r->pattern) == 0) { + /* match all or start on dir boundary */ + if (len == plen || + path[len - plen - 1] == '/') + return rule_matched(r); + } + } + } else { + if (r->fileonly) { + p = basename; + } else if (r->anchored || r->numseg == -1) { + /* full path matching */ + p = path; + } else { + short nseg = 1; + + /* match against the last numseg elements */ + for (p = path; *p != '\0'; p++) + if (*p == '/') + nseg++; + if (nseg < r->numseg) { + p = NULL; + } else { + nseg -= r->numseg; + for (p = path; *p != '\0' && nseg > 0; + p++) { + if (*p == '/') + nseg--; + } + } + } + + if (p != NULL) { + if (rmatch(r->pattern, p, r->leadingdir) == 0) + return rule_matched(r); + } + } + } + + return 0; +} Index: usr.bin/rsync/sender.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/sender.c,v retrieving revision 1.28 diff -u -p -u -r1.28 sender.c --- usr.bin/rsync/sender.c 5 Apr 2021 18:17:37 -0000 1.28 +++ usr.bin/rsync/sender.c 6 Nov 2021 18:35:18 -0000 @@ -358,7 +358,7 @@ rsync_sender(struct sess *sess, int fdin { struct flist *fl = NULL; const struct flist *f; - size_t i, flsz = 0, phase = 0, excl; + size_t i, flsz = 0, phase = 0; int rc = 0, c; int32_t idx; struct pollfd pfd[3]; @@ -393,12 +393,8 @@ rsync_sender(struct sess *sess, int fdin } /* Client sends zero-length exclusions if deleting. */ - - if (!sess->opts->server && sess->opts->del && - !io_write_int(sess, fdout, 0)) { - ERRX1("io_write_int"); - goto out; - } + if (!sess->opts->server && sess->opts->del) + send_rules(sess, fdout); /* * Then the file list in any mode. @@ -427,15 +423,8 @@ rsync_sender(struct sess *sess, int fdin * This is always 0 for now. */ - if (sess->opts->server) { - if (!io_read_size(sess, fdin, &excl)) { - ERRX1("io_read_size"); - goto out; - } else if (excl != 0) { - ERRX1("exclusion list is non-empty"); - goto out; - } - } + if (sess->opts->server) + recv_rules(sess, fdin); /* * Set up our poll events. Index: usr.bin/rsync/server.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/server.c,v retrieving revision 1.12 diff -u -p -u -r1.12 server.c --- usr.bin/rsync/server.c 8 May 2019 21:30:11 -0000 1.12 +++ usr.bin/rsync/server.c 6 Nov 2021 18:36:27 -0000 @@ -57,7 +57,7 @@ rsync_server(const struct opts *opts, si if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw unveil", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); memset(&sess, 0, sizeof(struct sess)); sess.opts = opts; @@ -65,7 +65,7 @@ rsync_server(const struct opts *opts, si /* Begin by making descriptors non-blocking. */ if (!fcntl_nonblock(fdin) || - !fcntl_nonblock(fdout)) { + !fcntl_nonblock(fdout)) { ERRX1("fcntl_nonblock"); goto out; } Index: usr.bin/rsync/session.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/session.c,v retrieving revision 1.7 diff -u -p -u -r1.7 session.c --- usr.bin/rsync/session.c 8 May 2019 20:00:25 -0000 1.7 +++ usr.bin/rsync/session.c 6 Nov 2021 18:36:42 -0000 @@ -14,7 +14,6 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include #include #include Index: usr.bin/rsync/socket.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/socket.c,v retrieving revision 1.29 diff -u -p -u -r1.29 socket.c --- usr.bin/rsync/socket.c 31 Mar 2021 19:45:16 -0000 1.29 +++ usr.bin/rsync/socket.c 6 Nov 2021 18:36:53 -0000 @@ -281,7 +281,7 @@ rsync_connect(const struct opts *opts, i if (pledge("stdio unix rpath wpath cpath dpath inet fattr chown dns getpw unveil", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); memset(&sess, 0, sizeof(struct sess)); sess.opts = opts; @@ -365,7 +365,7 @@ rsync_socket(const struct opts *opts, in if (pledge("stdio unix rpath wpath cpath dpath fattr chown getpw unveil", NULL) == -1) - err(1, "pledge"); + err(ERR_IPC, "pledge"); memset(&sess, 0, sizeof(struct sess)); sess.lver = RSYNC_PROTOCOL; @@ -374,10 +374,7 @@ rsync_socket(const struct opts *opts, in assert(f->host != NULL); assert(f->module != NULL); - if ((args = fargs_cmdline(&sess, f, &skip)) == NULL) { - ERRX1("fargs_cmdline"); - exit(1); - } + args = fargs_cmdline(&sess, f, &skip); /* Initiate with the rsyncd version and module request. */ Index: usr.bin/rsync/symlinks.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/symlinks.c,v retrieving revision 1.5 diff -u -p -u -r1.5 symlinks.c --- usr.bin/rsync/symlinks.c 8 May 2019 21:30:11 -0000 1.5 +++ usr.bin/rsync/symlinks.c 6 Nov 2021 18:28:00 -0000 @@ -14,11 +14,11 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include #include #include #include +#include #include #include "extern.h" @@ -36,7 +36,7 @@ symlink_read(const char *path) ssize_t nsz = 0; void *pp; - for (sz = MAXPATHLEN; ; sz *= 2) { + for (sz = PATH_MAX; ; sz *= 2) { if ((pp = realloc(buf, sz + 1)) == NULL) { ERR("realloc"); free(buf); @@ -75,7 +75,7 @@ symlinkat_read(int fd, const char *path) ssize_t nsz = 0; void *pp; - for (sz = MAXPATHLEN; ; sz *= 2) { + for (sz = PATH_MAX; ; sz *= 2) { if ((pp = realloc(buf, sz + 1)) == NULL) { ERR("realloc"); free(buf); Index: usr.bin/rsync/uploader.c =================================================================== RCS file: /cvs/src/usr.bin/rsync/uploader.c,v retrieving revision 1.24 diff -u -p -u -r1.24 uploader.c --- usr.bin/rsync/uploader.c 22 Mar 2021 11:20:04 -0000 1.24 +++ usr.bin/rsync/uploader.c 6 Nov 2021 18:32:47 -0000 @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -33,8 +34,7 @@ enum uploadst { UPLOAD_FIND_NEXT = 0, /* find next to upload to sender */ - UPLOAD_WRITE_LOCAL, /* wait to write to sender */ - UPLOAD_READ_LOCAL, /* wait to read from local file */ + UPLOAD_WRITE, /* wait to write to sender */ UPLOAD_FINISHED /* nothing more to do in phase */ }; @@ -80,7 +80,7 @@ log_dir(struct sess *sess, const struct * operator that we're a link. */ static void -log_link(struct sess *sess, const struct flist *f) +log_symlink(struct sess *sess, const struct flist *f) { if (!sess->opts->server) @@ -167,7 +167,7 @@ init_blk(struct blk *p, const struct blk * Return <0 on failure 0 on success. */ static int -pre_link(struct upload *p, struct sess *sess) +pre_symlink(struct upload *p, struct sess *sess) { struct stat st; const struct flist *f; @@ -180,8 +180,9 @@ pre_link(struct upload *p, struct sess * if (!sess->opts->preserve_links) { WARNX("%s: ignoring symlink", f->path); return 0; - } else if (sess->opts->dry_run) { - log_link(sess, f); + } + if (sess->opts->dry_run) { + log_symlink(sess, f); return 0; } @@ -194,6 +195,11 @@ pre_link(struct upload *p, struct sess * assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !S_ISLNK(st.st_mode)) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { @@ -201,9 +207,6 @@ pre_link(struct upload *p, struct sess * return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } /* @@ -259,12 +262,12 @@ pre_link(struct upload *p, struct sess * free(temp); } - log_link(sess, f); + log_symlink(sess, f); return 0; } /* - * See pre_link(), but for devices. + * See pre_symlink(), but for devices. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ @@ -282,7 +285,8 @@ pre_dev(struct upload *p, struct sess *s if (!sess->opts->devices || getuid() != 0) { WARNX("skipping non-regular file %s", f->path); return 0; - } else if (sess->opts->dry_run) { + } + if (sess->opts->dry_run) { log_file(sess, f); return 0; } @@ -296,6 +300,10 @@ pre_dev(struct upload *p, struct sess *s assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !(S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { @@ -303,9 +311,6 @@ pre_dev(struct upload *p, struct sess *s return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } /* Make sure existing device is of the correct type. */ @@ -351,7 +356,7 @@ pre_dev(struct upload *p, struct sess *s } /* - * See pre_link(), but for FIFOs. + * See pre_symlink(), but for FIFOs. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ @@ -369,7 +374,8 @@ pre_fifo(struct upload *p, struct sess * if (!sess->opts->specials) { WARNX("skipping non-regular file %s", f->path); return 0; - } else if (sess->opts->dry_run) { + } + if (sess->opts->dry_run) { log_file(sess, f); return 0; } @@ -383,6 +389,10 @@ pre_fifo(struct upload *p, struct sess * assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !S_ISFIFO(st.st_mode)) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { @@ -390,9 +400,6 @@ pre_fifo(struct upload *p, struct sess * return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } if (rc == -1) { @@ -426,7 +433,7 @@ pre_fifo(struct upload *p, struct sess * } /* - * See pre_link(), but for socket files. + * See pre_symlink(), but for socket files. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ @@ -444,7 +451,8 @@ pre_sock(struct upload *p, struct sess * if (!sess->opts->specials) { WARNX("skipping non-regular file %s", f->path); return 0; - } else if (sess->opts->dry_run) { + } + if (sess->opts->dry_run) { log_file(sess, f); return 0; } @@ -458,6 +466,10 @@ pre_sock(struct upload *p, struct sess * assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !S_ISSOCK(st.st_mode)) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { @@ -465,9 +477,6 @@ pre_sock(struct upload *p, struct sess * return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } if (rc == -1) { @@ -518,7 +527,8 @@ pre_dir(const struct upload *p, struct s if (!sess->opts->recursive) { WARNX("%s: ignoring directory", f->path); return 0; - } else if (sess->opts->dry_run) { + } + if (sess->opts->dry_run) { log_dir(sess, f); return 0; } @@ -529,7 +539,8 @@ pre_dir(const struct upload *p, struct s if (rc == -1 && errno != ENOENT) { ERR("%s: fstatat", f->path); return -1; - } else if (rc != -1 && !S_ISDIR(st.st_mode)) { + } + if (rc != -1 && !S_ISDIR(st.st_mode)) { ERRX("%s: not a directory", f->path); return -1; } else if (rc != -1) { @@ -579,13 +590,14 @@ post_dir(struct sess *sess, const struct if (!sess->opts->recursive) return 1; - else if (sess->opts->dry_run) + if (sess->opts->dry_run) return 1; if (fstatat(u->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW) == -1) { ERR("%s: fstatat", f->path); return 0; - } else if (!S_ISDIR(st.st_mode)) { + } + if (!S_ISDIR(st.st_mode)) { WARNX("%s: not a directory", f->path); return 0; } @@ -630,15 +642,55 @@ post_dir(struct sess *sess, const struct } /* + * Check if file exists in the specified root directory. + * Returns: + * -1 on error + * 0 if file is considered the same + * 1 if file exists and is possible match + * 2 if file exists but quick check failed + * 3 if file does not exist + * The stat pointer st is only valid for 0, 1, and 2 returns. + */ +static int +check_file(int rootfd, const struct flist *f, struct stat *st) +{ + if (fstatat(rootfd, f->path, st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) + return 3; + + ERR("%s: fstatat", f->path); + return -1; + } + + /* non-regular file needs attention */ + if (!S_ISREG(st->st_mode)) + return 2; + + /* quick check if file is the same */ + /* TODO: add support for --checksum, --size-only and --ignore-times */ + if (st->st_size == f->st.size) { + if (st->st_mtime == f->st.mtime) + return 0; + return 1; + } + + /* file needs attention */ + return 2; +} + +/* * Try to open the file at the current index. - * If the file does not exist, returns with success. + * If the file does not exist, returns with >0. * Return <0 on failure, 0 on success w/nothing to be done, >0 on * success and the file needs attention. */ static int -pre_file(const struct upload *p, int *filefd, struct sess *sess) +pre_file(const struct upload *p, int *filefd, off_t *size, + struct sess *sess) { const struct flist *f; + struct stat st; + int i, rc, match = -1; f = &p->fl[p->idx]; assert(S_ISREG(f->st.mode)); @@ -652,21 +704,90 @@ pre_file(const struct upload *p, int *fi return 0; } + if (sess->opts->max_size >= 0 && f->st.size > sess->opts->max_size) { + WARNX("skipping over max-size file %s", f->path); + return 0; + } + if (sess->opts->min_size >= 0 && f->st.size < sess->opts->min_size) { + WARNX("skipping under min-size file %s", f->path); + return 0; + } + /* * For non dry-run cases, we'll write the acknowledgement later - * in the rsync_uploader() function because we need to wait for - * the open() call to complete. - * If the call to openat() fails with ENOENT, there's a - * fast-path between here and the write function, so we won't do - * any blocking between now and then. + * in the rsync_uploader() function. */ - *filefd = openat(p->rootfd, f->path, - O_RDONLY | O_NOFOLLOW | O_NONBLOCK, 0); - if (*filefd != -1 || errno == ENOENT) - return 1; - ERR("%s: openat", f->path); - return -1; + *size = 0; + *filefd = -1; + + rc = check_file(p->rootfd, f, &st); + if (rc == -1) + return -1; + if (rc == 2 && !S_ISREG(st.st_mode)) { + if (S_ISDIR(st.st_mode) && + unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { + ERR("%s: unlinkat", f->path); + return -1; + } + } + if (rc == 0) { + if (!rsync_set_metadata_at(sess, 0, p->rootfd, f, f->path)) { + ERRX1("rsync_set_metadata"); + return -1; + } + LOG3("%s: skipping: up to date", f->path); + return 0; + } + + /* check alternative locations for better match */ + for (i = 0; sess->opts->basedir[i] != NULL; i++) { + const char *root = sess->opts->basedir[i]; + int dfd, x; + + dfd = openat(p->rootfd, root, O_RDONLY | O_DIRECTORY); + if (dfd == -1) + err(ERR_FILE_IO, "%s: openat", root); + x = check_file(dfd, f, &st); + /* found a match */ + if (x == 0) { + if (rc >= 0) { + /* found better match, delete file in rootfd */ + if (unlinkat(p->rootfd, f->path, 0) == -1 && + errno != ENOENT) { + ERR("%s: unlinkat", f->path); + return -1; + } + } + LOG3("%s: skipping: up to date in %s", f->path, root); + /* TODO: depending on mode link or copy file */ + close(dfd); + return 0; + } else if (x == 1 && match == -1) { + /* found a local file that is a close match */ + match = i; + } + close(dfd); + } + if (match != -1) { + /* copy match from basedir into root as a start point */ + copy_file(p->rootfd, sess->opts->basedir[match], f); + if (fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW) == + -1) { + ERR("%s: fstatat", f->path); + return -1; + } + } + + *size = st.st_size; + *filefd = openat(p->rootfd, f->path, O_RDONLY | O_NOFOLLOW); + if (*filefd == -1 && errno != ENOENT) { + ERR("%s: openat", f->path); + return -1; + } + + /* file needs attention */ + return 1; } /* @@ -737,16 +858,13 @@ rsync_uploader(struct upload *u, int *fi struct sess *sess, int *fileoutfd) { struct blkset blk; - struct stat st; void *mbuf, *bufp; ssize_t msz; size_t i, pos, sz; - off_t offs; + off_t offs, filesize; int c; - const struct flist *f; - - /* This should never get called. */ + /* Once finished this should never get called again. */ assert(u->state != UPLOAD_FINISHED); /* @@ -756,7 +874,7 @@ rsync_uploader(struct upload *u, int *fi * have a valid buffer to write. */ - if (u->state == UPLOAD_WRITE_LOCAL) { + if (u->state == UPLOAD_WRITE) { assert(u->buf != NULL); assert(*fileoutfd != -1); assert(*fileinfd == -1); @@ -810,9 +928,9 @@ rsync_uploader(struct upload *u, int *fi if (S_ISDIR(u->fl[u->idx].st.mode)) c = pre_dir(u, sess); else if (S_ISLNK(u->fl[u->idx].st.mode)) - c = pre_link(u, sess); + c = pre_symlink(u, sess); else if (S_ISREG(u->fl[u->idx].st.mode)) - c = pre_file(u, fileinfd, sess); + c = pre_file(u, fileinfd, &filesize, sess); else if (S_ISBLK(u->fl[u->idx].st.mode) || S_ISCHR(u->fl[u->idx].st.mode)) c = pre_dev(u, sess); @@ -848,66 +966,17 @@ rsync_uploader(struct upload *u, int *fi /* Go back to the event loop, if necessary. */ - u->state = (*fileinfd == -1) ? - UPLOAD_WRITE_LOCAL : UPLOAD_READ_LOCAL; - if (u->state == UPLOAD_READ_LOCAL) - return 1; - } - - /* - * If an input file is open, stat it and see if it's already up - * to date, in which case close it and go to the next one. - * Either way, we don't have a write channel open. - */ - - if (u->state == UPLOAD_READ_LOCAL) { - assert(*fileinfd != -1); - assert(*fileoutfd == -1); - f = &u->fl[u->idx]; - - if (fstat(*fileinfd, &st) == -1) { - ERR("%s: fstat", f->path); - close(*fileinfd); - *fileinfd = -1; - return -1; - } else if (!S_ISREG(st.st_mode)) { - ERRX("%s: not regular", f->path); - close(*fileinfd); - *fileinfd = -1; - return -1; - } - - if (st.st_size == f->st.size && - st.st_mtime == f->st.mtime) { - LOG3("%s: skipping: up to date", f->path); - if (!rsync_set_metadata - (sess, 0, *fileinfd, f, f->path)) { - ERRX1("rsync_set_metadata"); - close(*fileinfd); - *fileinfd = -1; - return -1; - } - close(*fileinfd); - *fileinfd = -1; - *fileoutfd = u->fdout; - u->state = UPLOAD_FIND_NEXT; - u->idx++; - return 1; - } - - /* Fallthrough... */ - - u->state = UPLOAD_WRITE_LOCAL; + u->state = UPLOAD_WRITE; } /* Initialies our blocks. */ - assert(u->state == UPLOAD_WRITE_LOCAL); + assert(u->state == UPLOAD_WRITE); memset(&blk, 0, sizeof(struct blkset)); blk.csum = u->csumlen; - if (*fileinfd != -1 && st.st_size > 0) { - init_blkset(&blk, st.st_size); + if (*fileinfd != -1 && filesize > 0) { + init_blkset(&blk, filesize); assert(blk.blksz); blk.blks = calloc(blk.blksz, sizeof(struct blk)); @@ -918,10 +987,11 @@ rsync_uploader(struct upload *u, int *fi return -1; } - if ((mbuf = calloc(1, blk.len)) == NULL) { - ERR("calloc"); + if ((mbuf = malloc(blk.len)) == NULL) { + ERR("malloc"); close(*fileinfd); *fileinfd = -1; + free(blk.blks); return -1; } @@ -929,16 +999,14 @@ rsync_uploader(struct upload *u, int *fi i = 0; do { msz = pread(*fileinfd, mbuf, blk.len, offs); - if (msz < 0) { + if ((size_t)msz != blk.len && (size_t)msz != blk.rem) { ERR("pread"); close(*fileinfd); *fileinfd = -1; + free(mbuf); + free(blk.blks); return -1; } - if ((size_t)msz != blk.len && (size_t)msz != blk.rem) { - /* short read, try again */ - continue; - } init_blk(&blk.blks[i], &blk, offs, i, mbuf, sess); offs += blk.len; LOG3( @@ -947,6 +1015,7 @@ rsync_uploader(struct upload *u, int *fi i++; } while (i < blk.blksz); + free(mbuf); close(*fileinfd); *fileinfd = -1; LOG3("%s: mapped %jd B with %zu blocks", @@ -966,18 +1035,19 @@ rsync_uploader(struct upload *u, int *fi /* Make sure the block metadata buffer is big enough. */ u->bufsz = - sizeof(int32_t) + /* identifier */ - sizeof(int32_t) + /* block count */ - sizeof(int32_t) + /* block length */ - sizeof(int32_t) + /* checksum length */ - sizeof(int32_t) + /* block remainder */ - blk.blksz * - (sizeof(int32_t) + /* short checksum */ - blk.csum); /* long checksum */ + sizeof(int32_t) + /* identifier */ + sizeof(int32_t) + /* block count */ + sizeof(int32_t) + /* block length */ + sizeof(int32_t) + /* checksum length */ + sizeof(int32_t) + /* block remainder */ + blk.blksz * + (sizeof(int32_t) + /* short checksum */ + blk.csum); /* long checksum */ if (u->bufsz > u->bufmax) { if ((bufp = realloc(u->buf, u->bufsz)) == NULL) { ERR("realloc"); + free(blk.blks); return -1; } u->buf = bufp;