1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3 #include <curl/curl.h>
4 #include <sys/prctl.h>
5
6 #include "sd-daemon.h"
7
8 #include "alloc-util.h"
9 #include "btrfs-util.h"
10 #include "copy.h"
11 #include "curl-util.h"
12 #include "fd-util.h"
13 #include "fs-util.h"
14 #include "hostname-util.h"
15 #include "import-common.h"
16 #include "import-util.h"
17 #include "install-file.h"
18 #include "macro.h"
19 #include "mkdir-label.h"
20 #include "path-util.h"
21 #include "process-util.h"
22 #include "pull-common.h"
23 #include "pull-job.h"
24 #include "pull-tar.h"
25 #include "rm-rf.h"
26 #include "string-util.h"
27 #include "strv.h"
28 #include "tmpfile-util.h"
29 #include "user-util.h"
30 #include "utf8.h"
31 #include "util.h"
32 #include "web-util.h"
33
34 typedef enum TarProgress {
35 TAR_DOWNLOADING,
36 TAR_VERIFYING,
37 TAR_FINALIZING,
38 TAR_COPYING,
39 } TarProgress;
40
41 struct TarPull {
42 sd_event *event;
43 CurlGlue *glue;
44
45 PullFlags flags;
46 ImportVerify verify;
47 char *image_root;
48
49 PullJob *tar_job;
50 PullJob *checksum_job;
51 PullJob *signature_job;
52 PullJob *settings_job;
53
54 TarPullFinished on_finished;
55 void *userdata;
56
57 char *local;
58
59 pid_t tar_pid;
60
61 char *final_path;
62 char *temp_path;
63
64 char *settings_path;
65 char *settings_temp_path;
66
67 char *checksum;
68 };
69
tar_pull_unref(TarPull * i)70 TarPull* tar_pull_unref(TarPull *i) {
71 if (!i)
72 return NULL;
73
74 if (i->tar_pid > 1)
75 sigkill_wait(i->tar_pid);
76
77 pull_job_unref(i->tar_job);
78 pull_job_unref(i->checksum_job);
79 pull_job_unref(i->signature_job);
80 pull_job_unref(i->settings_job);
81
82 curl_glue_unref(i->glue);
83 sd_event_unref(i->event);
84
85 rm_rf_subvolume_and_free(i->temp_path);
86 unlink_and_free(i->settings_temp_path);
87
88 free(i->final_path);
89 free(i->settings_path);
90 free(i->image_root);
91 free(i->local);
92 free(i->checksum);
93
94 return mfree(i);
95 }
96
tar_pull_new(TarPull ** ret,sd_event * event,const char * image_root,TarPullFinished on_finished,void * userdata)97 int tar_pull_new(
98 TarPull **ret,
99 sd_event *event,
100 const char *image_root,
101 TarPullFinished on_finished,
102 void *userdata) {
103
104 _cleanup_(curl_glue_unrefp) CurlGlue *g = NULL;
105 _cleanup_(sd_event_unrefp) sd_event *e = NULL;
106 _cleanup_(tar_pull_unrefp) TarPull *i = NULL;
107 _cleanup_free_ char *root = NULL;
108 int r;
109
110 assert(ret);
111
112 root = strdup(image_root ?: "/var/lib/machines");
113 if (!root)
114 return -ENOMEM;
115
116 if (event)
117 e = sd_event_ref(event);
118 else {
119 r = sd_event_default(&e);
120 if (r < 0)
121 return r;
122 }
123
124 r = curl_glue_new(&g, e);
125 if (r < 0)
126 return r;
127
128 i = new(TarPull, 1);
129 if (!i)
130 return -ENOMEM;
131
132 *i = (TarPull) {
133 .on_finished = on_finished,
134 .userdata = userdata,
135 .image_root = TAKE_PTR(root),
136 .event = TAKE_PTR(e),
137 .glue = TAKE_PTR(g),
138 };
139
140 i->glue->on_finished = pull_job_curl_on_finished;
141 i->glue->userdata = i;
142
143 *ret = TAKE_PTR(i);
144
145 return 0;
146 }
147
tar_pull_report_progress(TarPull * i,TarProgress p)148 static void tar_pull_report_progress(TarPull *i, TarProgress p) {
149 unsigned percent;
150
151 assert(i);
152
153 switch (p) {
154
155 case TAR_DOWNLOADING: {
156 unsigned remain = 85;
157
158 percent = 0;
159
160 if (i->checksum_job) {
161 percent += i->checksum_job->progress_percent * 5 / 100;
162 remain -= 5;
163 }
164
165 if (i->signature_job) {
166 percent += i->signature_job->progress_percent * 5 / 100;
167 remain -= 5;
168 }
169
170 if (i->settings_job) {
171 percent += i->settings_job->progress_percent * 5 / 100;
172 remain -= 5;
173 }
174
175 if (i->tar_job)
176 percent += i->tar_job->progress_percent * remain / 100;
177 break;
178 }
179
180 case TAR_VERIFYING:
181 percent = 85;
182 break;
183
184 case TAR_FINALIZING:
185 percent = 90;
186 break;
187
188 case TAR_COPYING:
189 percent = 95;
190 break;
191
192 default:
193 assert_not_reached();
194 }
195
196 sd_notifyf(false, "X_IMPORT_PROGRESS=%u", percent);
197 log_debug("Combined progress %u%%", percent);
198 }
199
tar_pull_determine_path(TarPull * i,const char * suffix,char ** field)200 static int tar_pull_determine_path(
201 TarPull *i,
202 const char *suffix,
203 char **field /* input + output (!) */) {
204 int r;
205
206 assert(i);
207 assert(field);
208
209 if (*field)
210 return 0;
211
212 assert(i->tar_job);
213
214 r = pull_make_path(i->tar_job->url, i->tar_job->etag, i->image_root, ".tar-", suffix, field);
215 if (r < 0)
216 return log_oom();
217
218 return 1;
219 }
220
tar_pull_make_local_copy(TarPull * i)221 static int tar_pull_make_local_copy(TarPull *i) {
222 _cleanup_(rm_rf_subvolume_and_freep) char *t = NULL;
223 const char *p;
224 int r;
225
226 assert(i);
227 assert(i->tar_job);
228
229 if (!i->local)
230 return 0;
231
232 assert(i->final_path);
233
234 p = prefix_roota(i->image_root, i->local);
235
236 r = tempfn_random(p, NULL, &t);
237 if (r < 0)
238 return log_error_errno(r, "Failed to generate temporary filename for %s: %m", p);
239
240 if (i->flags & PULL_BTRFS_SUBVOL)
241 r = btrfs_subvol_snapshot(
242 i->final_path,
243 t,
244 (i->flags & PULL_BTRFS_QUOTA ? BTRFS_SNAPSHOT_QUOTA : 0)|
245 BTRFS_SNAPSHOT_FALLBACK_COPY|
246 BTRFS_SNAPSHOT_FALLBACK_DIRECTORY|
247 BTRFS_SNAPSHOT_RECURSIVE);
248 else
249 r = copy_tree(i->final_path, t, UID_INVALID, GID_INVALID, COPY_REFLINK|COPY_HARDLINKS);
250 if (r < 0)
251 return log_error_errno(r, "Failed to create local image: %m");
252
253 r = install_file(AT_FDCWD, t,
254 AT_FDCWD, p,
255 (i->flags & PULL_FORCE ? INSTALL_REPLACE : 0) |
256 (i->flags & PULL_READ_ONLY ? INSTALL_READ_ONLY : 0) |
257 (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
258 if (r < 0)
259 return log_error_errno(r, "Failed to install local image '%s': %m", p);
260
261 t = mfree(t);
262
263 log_info("Created new local image '%s'.", i->local);
264
265 if (FLAGS_SET(i->flags, PULL_SETTINGS)) {
266 const char *local_settings;
267 assert(i->settings_job);
268
269 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
270 if (r < 0)
271 return r;
272
273 local_settings = strjoina(i->image_root, "/", i->local, ".nspawn");
274
275 r = copy_file_atomic(
276 i->settings_path,
277 local_settings,
278 0664,
279 0, 0,
280 COPY_REFLINK |
281 (FLAGS_SET(i->flags, PULL_FORCE) ? COPY_REPLACE : 0) |
282 (FLAGS_SET(i->flags, PULL_SYNC) ? COPY_FSYNC_FULL : 0));
283 if (r == -EEXIST)
284 log_warning_errno(r, "Settings file %s already exists, not replacing.", local_settings);
285 else if (r == -ENOENT)
286 log_debug_errno(r, "Skipping creation of settings file, since none was found.");
287 else if (r < 0)
288 log_warning_errno(r, "Failed to copy settings files %s, ignoring: %m", local_settings);
289 else
290 log_info("Created new settings file %s.", local_settings);
291 }
292
293 return 0;
294 }
295
tar_pull_is_done(TarPull * i)296 static bool tar_pull_is_done(TarPull *i) {
297 assert(i);
298 assert(i->tar_job);
299
300 if (!PULL_JOB_IS_COMPLETE(i->tar_job))
301 return false;
302 if (i->checksum_job && !PULL_JOB_IS_COMPLETE(i->checksum_job))
303 return false;
304 if (i->signature_job && !PULL_JOB_IS_COMPLETE(i->signature_job))
305 return false;
306 if (i->settings_job && !PULL_JOB_IS_COMPLETE(i->settings_job))
307 return false;
308
309 return true;
310 }
311
tar_pull_job_on_finished(PullJob * j)312 static void tar_pull_job_on_finished(PullJob *j) {
313 TarPull *i;
314 int r;
315
316 assert(j);
317 assert(j->userdata);
318
319 i = j->userdata;
320
321 if (j->error != 0) {
322 if (j == i->tar_job) {
323 if (j->error == ENOMEDIUM) /* HTTP 404 */
324 r = log_error_errno(j->error, "Failed to retrieve image file. (Wrong URL?)");
325 else
326 r = log_error_errno(j->error, "Failed to retrieve image file.");
327 goto finish;
328 } else if (j == i->checksum_job) {
329 r = log_error_errno(j->error, "Failed to retrieve SHA256 checksum, cannot verify. (Try --verify=no?)");
330 goto finish;
331 } else if (j == i->signature_job)
332 log_debug_errno(j->error, "Signature job for %s failed, proceeding for now.", j->url);
333 else if (j == i->settings_job)
334 log_info_errno(j->error, "Settings file could not be retrieved, proceeding without.");
335 else
336 assert("unexpected job");
337 }
338
339 /* This is invoked if either the download completed successfully, or the download was skipped because
340 * we already have the etag. */
341
342 if (!tar_pull_is_done(i))
343 return;
344
345 if (i->signature_job && i->signature_job->error != 0) {
346 VerificationStyle style;
347
348 assert(i->checksum_job);
349
350 r = verification_style_from_url(i->checksum_job->url, &style);
351 if (r < 0) {
352 log_error_errno(r, "Failed to determine verification style from checksum URL: %m");
353 goto finish;
354 }
355
356 if (style == VERIFICATION_PER_DIRECTORY) { /* A failed signature file download only matters
357 * in per-directory verification mode, since only
358 * then the signature is detached, and thus a file
359 * of its own. */
360 r = log_error_errno(i->signature_job->error,
361 "Failed to retrieve signature file, cannot verify. (Try --verify=no?)");
362 goto finish;
363 }
364 }
365
366 pull_job_close_disk_fd(i->tar_job);
367 pull_job_close_disk_fd(i->settings_job);
368
369 if (i->tar_pid > 0) {
370 r = wait_for_terminate_and_check("tar", TAKE_PID(i->tar_pid), WAIT_LOG);
371 if (r < 0)
372 goto finish;
373 if (r != EXIT_SUCCESS) {
374 r = -EIO;
375 goto finish;
376 }
377 }
378
379 if (!i->tar_job->etag_exists) {
380 /* This is a new download, verify it, and move it into place */
381
382 tar_pull_report_progress(i, TAR_VERIFYING);
383
384 r = pull_verify(i->verify,
385 i->checksum,
386 i->tar_job,
387 i->checksum_job,
388 i->signature_job,
389 i->settings_job,
390 /* roothash_job = */ NULL,
391 /* roothash_signature_job = */ NULL,
392 /* verity_job = */ NULL);
393 if (r < 0)
394 goto finish;
395 }
396
397 if (i->flags & PULL_DIRECT) {
398 assert(!i->settings_job);
399 assert(i->local);
400 assert(!i->temp_path);
401
402 tar_pull_report_progress(i, TAR_FINALIZING);
403
404 r = import_mangle_os_tree(i->local);
405 if (r < 0)
406 goto finish;
407
408 r = install_file(
409 AT_FDCWD, i->local,
410 AT_FDCWD, NULL,
411 (i->flags & PULL_READ_ONLY) ? INSTALL_READ_ONLY : 0 |
412 (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
413 if (r < 0) {
414 log_error_errno(r, "Failed to finalize '%s': %m", i->local);
415 goto finish;
416 }
417 } else {
418 r = tar_pull_determine_path(i, NULL, &i->final_path);
419 if (r < 0)
420 goto finish;
421
422 if (!i->tar_job->etag_exists) {
423 /* This is a new download, verify it, and move it into place */
424
425 assert(i->temp_path);
426 assert(i->final_path);
427
428 tar_pull_report_progress(i, TAR_FINALIZING);
429
430 r = import_mangle_os_tree(i->temp_path);
431 if (r < 0)
432 goto finish;
433
434 r = install_file(
435 AT_FDCWD, i->temp_path,
436 AT_FDCWD, i->final_path,
437 INSTALL_READ_ONLY|
438 (i->flags & PULL_SYNC ? INSTALL_SYNCFS : 0));
439 if (r < 0) {
440 log_error_errno(r, "Failed to rename to final image name to %s: %m", i->final_path);
441 goto finish;
442 }
443
444 i->temp_path = mfree(i->temp_path);
445
446 if (i->settings_job &&
447 i->settings_job->error == 0) {
448
449 /* Also move the settings file into place, if it exists. Note that we do so only if we also
450 * moved the tar file in place, to keep things strictly in sync. */
451 assert(i->settings_temp_path);
452
453 /* Regenerate final name for this auxiliary file, we might know the etag of the file now, and
454 * we should incorporate it in the file name if we can */
455 i->settings_path = mfree(i->settings_path);
456
457 r = tar_pull_determine_path(i, ".nspawn", &i->settings_path);
458 if (r < 0)
459 goto finish;
460
461 r = install_file(
462 AT_FDCWD, i->settings_temp_path,
463 AT_FDCWD, i->settings_path,
464 INSTALL_READ_ONLY|
465 (i->flags & PULL_SYNC ? INSTALL_FSYNC_FULL : 0));
466 if (r < 0) {
467 log_error_errno(r, "Failed to rename settings file to %s: %m", i->settings_path);
468 goto finish;
469 }
470
471 i->settings_temp_path = mfree(i->settings_temp_path);
472 }
473 }
474
475 tar_pull_report_progress(i, TAR_COPYING);
476
477 r = tar_pull_make_local_copy(i);
478 if (r < 0)
479 goto finish;
480 }
481
482 r = 0;
483
484 finish:
485 if (i->on_finished)
486 i->on_finished(i, r, i->userdata);
487 else
488 sd_event_exit(i->event, r);
489 }
490
tar_pull_job_on_open_disk_tar(PullJob * j)491 static int tar_pull_job_on_open_disk_tar(PullJob *j) {
492 const char *where;
493 TarPull *i;
494 int r;
495
496 assert(j);
497 assert(j->userdata);
498
499 i = j->userdata;
500 assert(i->tar_job == j);
501 assert(i->tar_pid <= 0);
502
503 if (i->flags & PULL_DIRECT)
504 where = i->local;
505 else {
506 if (!i->temp_path) {
507 r = tempfn_random_child(i->image_root, "tar", &i->temp_path);
508 if (r < 0)
509 return log_oom();
510 }
511
512 where = i->temp_path;
513 }
514
515 (void) mkdir_parents_label(where, 0700);
516
517 if (FLAGS_SET(i->flags, PULL_DIRECT|PULL_FORCE))
518 (void) rm_rf(where, REMOVE_ROOT|REMOVE_PHYSICAL|REMOVE_SUBVOLUME);
519
520 if (i->flags & PULL_BTRFS_SUBVOL)
521 r = btrfs_subvol_make_fallback(where, 0755);
522 else
523 r = RET_NERRNO(mkdir(where, 0755));
524 if (r == -EEXIST && (i->flags & PULL_DIRECT)) /* EEXIST is OK if in direct mode, but not otherwise,
525 * because in that case our temporary path collided */
526 r = 0;
527 if (r < 0)
528 return log_error_errno(r, "Failed to create directory/subvolume %s: %m", where);
529 if (r > 0 && (i->flags & PULL_BTRFS_QUOTA)) { /* actually btrfs subvol */
530 if (!(i->flags & PULL_DIRECT))
531 (void) import_assign_pool_quota_and_warn(i->image_root);
532 (void) import_assign_pool_quota_and_warn(where);
533 }
534
535 j->disk_fd = import_fork_tar_x(where, &i->tar_pid);
536 if (j->disk_fd < 0)
537 return j->disk_fd;
538
539 return 0;
540 }
541
tar_pull_job_on_open_disk_settings(PullJob * j)542 static int tar_pull_job_on_open_disk_settings(PullJob *j) {
543 TarPull *i;
544 int r;
545
546 assert(j);
547 assert(j->userdata);
548
549 i = j->userdata;
550 assert(i->settings_job == j);
551
552 if (!i->settings_temp_path) {
553 r = tempfn_random_child(i->image_root, "settings", &i->settings_temp_path);
554 if (r < 0)
555 return log_oom();
556 }
557
558 (void) mkdir_parents_label(i->settings_temp_path, 0700);
559
560 j->disk_fd = open(i->settings_temp_path, O_RDWR|O_CREAT|O_EXCL|O_NOCTTY|O_CLOEXEC, 0664);
561 if (j->disk_fd < 0)
562 return log_error_errno(errno, "Failed to create %s: %m", i->settings_temp_path);
563
564 return 0;
565 }
566
tar_pull_job_on_progress(PullJob * j)567 static void tar_pull_job_on_progress(PullJob *j) {
568 TarPull *i;
569
570 assert(j);
571 assert(j->userdata);
572
573 i = j->userdata;
574
575 tar_pull_report_progress(i, TAR_DOWNLOADING);
576 }
577
tar_pull_start(TarPull * i,const char * url,const char * local,PullFlags flags,ImportVerify verify,const char * checksum)578 int tar_pull_start(
579 TarPull *i,
580 const char *url,
581 const char *local,
582 PullFlags flags,
583 ImportVerify verify,
584 const char *checksum) {
585
586 PullJob *j;
587 int r;
588
589 assert(i);
590 assert(verify == _IMPORT_VERIFY_INVALID || verify < _IMPORT_VERIFY_MAX);
591 assert(verify == _IMPORT_VERIFY_INVALID || verify >= 0);
592 assert((verify < 0) || !checksum);
593 assert(!(flags & ~PULL_FLAGS_MASK_TAR));
594 assert(!(flags & PULL_SETTINGS) || !(flags & PULL_DIRECT));
595 assert(!(flags & PULL_SETTINGS) || !checksum);
596
597 if (!http_url_is_valid(url) && !file_url_is_valid(url))
598 return -EINVAL;
599
600 if (local && !pull_validate_local(local, flags))
601 return -EINVAL;
602
603 if (i->tar_job)
604 return -EBUSY;
605
606 r = free_and_strdup(&i->local, local);
607 if (r < 0)
608 return r;
609
610 r = free_and_strdup(&i->checksum, checksum);
611 if (r < 0)
612 return r;
613
614 i->flags = flags;
615 i->verify = verify;
616
617 /* Set up download job for TAR file */
618 r = pull_job_new(&i->tar_job, url, i->glue, i);
619 if (r < 0)
620 return r;
621
622 i->tar_job->on_finished = tar_pull_job_on_finished;
623 i->tar_job->on_open_disk = tar_pull_job_on_open_disk_tar;
624 i->tar_job->calc_checksum = checksum || IN_SET(verify, IMPORT_VERIFY_CHECKSUM, IMPORT_VERIFY_SIGNATURE);
625
626 if (!FLAGS_SET(flags, PULL_DIRECT)) {
627 r = pull_find_old_etags(url, i->image_root, DT_DIR, ".tar-", NULL, &i->tar_job->old_etags);
628 if (r < 0)
629 return r;
630 }
631
632 /* Set up download of checksum/signature files */
633 r = pull_make_verification_jobs(
634 &i->checksum_job,
635 &i->signature_job,
636 verify,
637 checksum,
638 url,
639 i->glue,
640 tar_pull_job_on_finished,
641 i);
642 if (r < 0)
643 return r;
644
645 /* Set up download job for the settings file (.nspawn) */
646 if (FLAGS_SET(flags, PULL_SETTINGS)) {
647 r = pull_make_auxiliary_job(
648 &i->settings_job,
649 url,
650 tar_strip_suffixes,
651 ".nspawn",
652 verify,
653 i->glue,
654 tar_pull_job_on_open_disk_settings,
655 tar_pull_job_on_finished,
656 i);
657 if (r < 0)
658 return r;
659 }
660
661 FOREACH_POINTER(j,
662 i->tar_job,
663 i->checksum_job,
664 i->signature_job,
665 i->settings_job) {
666
667 if (!j)
668 continue;
669
670 j->on_progress = tar_pull_job_on_progress;
671 j->sync = FLAGS_SET(flags, PULL_SYNC);
672
673 r = pull_job_begin(j);
674 if (r < 0)
675 return r;
676 }
677
678 return 0;
679 }
680