liberasurecode 1.6.3
Erasure Code API library
Loading...
Searching...
No Matches
erasurecode.c
Go to the documentation of this file.
1/*
2 * Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert, Mark Storer
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions are met:
6 *
7 * Redistributions of source code must retain the above copyright notice, this
8 * list of conditions and the following disclaimer.
9 *
10 * Redistributions in binary form must reproduce the above copyright notice, this
11 * list of conditions and the following disclaimer in the documentation and/or
12 * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13 * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16 * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 *
24 * liberasurecode API implementation
25 *
26 * vi: set noai tw=79 ts=4 sw=4:
27 */
28
29#include <assert.h>
30#include <zlib.h>
31#include "list.h"
32#include "erasurecode.h"
33#include "erasurecode_backend.h"
34#include "erasurecode_helpers.h"
35#include "erasurecode_helpers_ext.h"
36#include "erasurecode_preprocessing.h"
37#include "erasurecode_postprocessing.h"
38#include "erasurecode_stdinc.h"
39
40#include "alg_sig.h"
41#include "erasurecode_log.h"
42
43/* =~=*=~==~=*=~==~=*=~= Supported EC backends =~=*=~==~=*=~==~=*=~==~=*=~== */
44
45/* EC backend references */
46extern struct ec_backend_common backend_null;
47extern struct ec_backend_common backend_flat_xor_hd;
48extern struct ec_backend_common backend_jerasure_rs_vand;
49extern struct ec_backend_common backend_jerasure_rs_cauchy;
50extern struct ec_backend_common backend_isa_l_rs_vand;
51extern struct ec_backend_common backend_shss;
52extern struct ec_backend_common backend_liberasurecode_rs_vand;
53extern struct ec_backend_common backend_isa_l_rs_cauchy;
54extern struct ec_backend_common backend_libphazr;
55
56ec_backend_t ec_backends_supported[] = {
57 (ec_backend_t) &backend_null,
58 (ec_backend_t) &backend_jerasure_rs_vand,
59 (ec_backend_t) &backend_jerasure_rs_cauchy,
60 (ec_backend_t) &backend_flat_xor_hd,
61 (ec_backend_t) &backend_isa_l_rs_vand,
62 (ec_backend_t) &backend_shss,
63 (ec_backend_t) &backend_liberasurecode_rs_vand,
64 (ec_backend_t) &backend_isa_l_rs_cauchy,
65 (ec_backend_t) &backend_libphazr,
66 NULL,
67};
68
69/* backend list to return to the caller */
71char *ec_backends_supported_str[EC_BACKENDS_MAX];
72
73/* =~=*=~==~=*=~==~=*=~= EC backend instance management =~=*=~==~=*=~==~=*= */
74
75/* Registered erasure code backend instances */
76SLIST_HEAD(backend_list, ec_backend) active_instances =
77 SLIST_HEAD_INITIALIZER(active_instances);
78rwlock_t active_instances_rwlock = RWLOCK_INITIALIZER;
79
80/* Backend instance id */
81int next_backend_desc = 0;
82
89ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc)
90{
91 struct ec_backend *b = NULL;
92 SLIST_FOREACH(b, &active_instances, link) {
93 if (b->idesc == desc)
94 break;
95 }
96 return b;
97}
98
106{
107 for (;;) {
108 if (++next_backend_desc <= 0)
109 next_backend_desc = 1;
110 if (!liberasurecode_backend_instance_get_by_desc(next_backend_desc))
111 return next_backend_desc;
112 }
113}
114
123{
124 int desc = -1; /* descriptor to return */
125 int rc = 0; /* return call value */
126
127 rc = rwlock_wrlock(&active_instances_rwlock);
128 if (rc == 0) {
129 SLIST_INSERT_HEAD(&active_instances, instance, link);
131 if (desc <= 0)
132 goto register_out;
133 instance->idesc = desc;
134 } else {
135 goto exit;
136 }
137
138register_out:
139 rwlock_unlock(&active_instances_rwlock);
140exit:
141 return desc;
142}
143
150{
151 int rc = 0; /* return call value */
152
153 rc = rwlock_wrlock(&active_instances_rwlock);
154 if (rc == 0) {
155 SLIST_REMOVE(&active_instances, instance, ec_backend, link);
156 } else {
157 goto exit;
158 }
159 rwlock_unlock(&active_instances_rwlock);
160
161exit:
162 return rc;
163}
164
165/* =~=*=~==~=*=~== liberasurecode backend API helpers =~=*=~==~=*=~== */
166
167static void print_dlerror(const char *caller)
168{
169 char *msg = dlerror();
170 if (NULL == msg)
171 log_error("%s: unknown dynamic linking error\n", caller);
172 else
173 log_error("%s: dynamic linking error %s\n", caller, msg);
174}
175
176/* Generic dlopen/dlclose routines */
177void* liberasurecode_backend_open(ec_backend_t instance)
178{
179 if (NULL == instance)
180 return NULL;
181 /* Use RTLD_LOCAL to avoid symbol collisions */
182 return dlopen(instance->common.soname, RTLD_LAZY | RTLD_LOCAL);
183}
184
185int liberasurecode_backend_close(ec_backend_t instance)
186{
187 if (NULL == instance || NULL == instance->desc.backend_sohandle)
188 return 0;
189
190 dlclose(instance->desc.backend_sohandle);
191 dlerror(); /* Clear any existing errors */
192
193 instance->desc.backend_sohandle = NULL;
194 return 0;
195}
196
197/* =*=~==~=*=~==~=*=~= liberasurecode init/exit routines =~=*=~==~=*=~==~=*= */
198
199void __attribute__ ((constructor))
200liberasurecode_init(void) {
201 /* init logging */
202 openlog("liberasurecode", LOG_PID | LOG_CONS, LOG_USER);
203
204 /* populate supported backends list as a string */
205 {
206 int i;
207 for (i = 0; ec_backends_supported[i]; ++i) {
208 ec_backends_supported_str[i] = strdup(
209 ec_backends_supported[i]->common.name);
210 }
212 }
213}
214
215void __attribute__ ((destructor))
216liberasurecode_exit(void) {
217 int i;
218 for (i = 0; i < num_supported_backends; ++i)
220 closelog();
221}
222
223/* =~=*=~==~=*=~= liberasurecode frontend API implementation =~=*=~==~=*=~== */
224
232int liberasurecode_backend_available(const ec_backend_id_t backend_id) {
233 struct ec_backend backend;
234 if (backend_id >= EC_BACKENDS_MAX)
235 return 0;
236
237 backend.desc.backend_sohandle = liberasurecode_backend_open(
238 ec_backends_supported[backend_id]);
239 if (!backend.desc.backend_sohandle) {
240 return 0;
241 }
242
244 return 1;
245}
246
266int liberasurecode_instance_create(const ec_backend_id_t id,
267 struct ec_args *args)
268{
269 ec_backend_t instance = NULL;
270 struct ec_backend_args bargs;
271 if (!args)
272 return -EINVALIDPARAMS;
273
274 if (id >= EC_BACKENDS_MAX)
275 return -EBACKENDNOTSUPP;
276
277 if (args->k < 0 || args->m < 0)
278 return -EINVALIDPARAMS;
279 if ((args->k + args->m) > EC_MAX_FRAGMENTS) {
280 log_error("Total number of fragments (k + m) must be less than %d\n",
281 EC_MAX_FRAGMENTS);
282 return -EINVALIDPARAMS;
283 }
284
285 /* Allocate memory for ec_backend instance */
286 instance = calloc(1, sizeof(*instance));
287 if (NULL == instance)
288 return -ENOMEM;
289
290 /* Copy common backend, args struct */
291 instance->common = ec_backends_supported[id]->common;
292 memcpy(&(bargs.uargs), args, sizeof (struct ec_args));
293 instance->args = bargs;
294
295 /* Open backend .so if not already open */
296 /* .so handle is returned in instance->desc.backend_sohandle */
297 if (!instance->desc.backend_sohandle) {
298 instance->desc.backend_sohandle = liberasurecode_backend_open(instance);
299 if (!instance->desc.backend_sohandle) {
300 /* ignore during init, return the same handle */
301 print_dlerror(__func__);
302 free(instance);
303 return -EBACKENDNOTAVAIL;
304 }
305 }
306
307 /* Call private init() for the backend */
308 instance->desc.backend_desc = instance->common.ops->init(
309 &instance->args, instance->desc.backend_sohandle);
310 if (NULL == instance->desc.backend_desc) {
311 free (instance);
312 return -EBACKENDINITERR;
313 }
314
315 /* Register instance and return a descriptor/instance id */
316 instance->idesc = liberasurecode_backend_instance_register(instance);
317
318 return instance->idesc;
319}
320
327{
328 ec_backend_t instance = NULL; /* instance to destroy */
329 int rc = 0; /* return code */
330
331 instance = liberasurecode_backend_instance_get_by_desc(desc);
332 if (NULL == instance)
333 return -EBACKENDNOTAVAIL;
334
335 /* Call private exit() for the backend */
336 instance->common.ops->exit(instance->desc.backend_desc);
337
338 /* dlclose() backend library */
340
341 /* Remove instance from registry */
343 if (rc == 0) {
344 free(instance);
345 }
346
347 return rc;
348}
349
366 char **encoded_data,
367 char **encoded_parity)
368{
369 int i, k, m;
370
371 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
372 if (NULL == instance) {
373 return -EBACKENDNOTAVAIL;
374 }
375
376 k = instance->args.uargs.k;
377 m = instance->args.uargs.m;
378
379 if (encoded_data) {
380 for (i = 0; i < k; i++) {
381 free(encoded_data[i]);
382 }
383
384 free(encoded_data);
385 }
386
387 if (encoded_parity) {
388 for (i = 0; i < m; i++) {
389 free(encoded_parity[i]);
390 }
391 free(encoded_parity);
392 }
393
394 return 0;
395}
396
414 const char *orig_data, uint64_t orig_data_size, /* input */
415 char ***encoded_data, char ***encoded_parity, /* output */
416 uint64_t *fragment_len) /* output */
417{
418 int k, m;
419 int ret = 0; /* return code */
420
421 int blocksize = 0; /* length of each of k data elements */
422
423 if (orig_data == NULL) {
424 log_error("Pointer to data buffer is null!");
425 ret = -EINVALIDPARAMS;
426 goto out;
427 }
428
429 if (encoded_data == NULL) {
430 log_error("Pointer to encoded data buffers is null!");
431 return -EINVALIDPARAMS;
432 }
433
434 if (encoded_parity == NULL) {
435 log_error("Pointer to encoded parity buffers is null!");
436 return -EINVALIDPARAMS;
437 }
438
439 if (fragment_len == NULL) {
440 log_error("Pointer to fragment length is null!");
441 ret = -EINVALIDPARAMS;
442 goto out;
443 }
444
445 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
446 if (NULL == instance) {
447 ret = -EBACKENDNOTAVAIL;
448 goto out;
449 }
450
451 k = instance->args.uargs.k;
452 m = instance->args.uargs.m;
453
454 /*
455 * Allocate arrays for data, parity and missing_idxs
456 */
457 *encoded_data = (char **) alloc_zeroed_buffer(sizeof(char *) * k);
458 if (NULL == *encoded_data) {
459 log_error("Could not allocate data buffer!");
460 goto out;
461 }
462
463 *encoded_parity = (char **) alloc_zeroed_buffer(sizeof(char *) * m);
464 if (NULL == *encoded_parity) {
465 log_error("Could not allocate parity buffer!");
466 goto out;
467 }
468
469 ret = prepare_fragments_for_encode(instance, k, m, orig_data, orig_data_size,
470 *encoded_data, *encoded_parity, &blocksize);
471 if (ret < 0) {
472 // ensure encoded_data/parity point the head of fragment_ptr
473 get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
474 get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
475 goto out;
476 }
477
478 /* call the backend encode function passing it desc instance */
479 ret = instance->common.ops->encode(instance->desc.backend_desc,
480 *encoded_data, *encoded_parity, blocksize);
481 if (ret < 0) {
482 // ensure encoded_data/parity point the head of fragment_ptr
483 get_fragment_ptr_array_from_data(*encoded_data, *encoded_data, k);
484 get_fragment_ptr_array_from_data(*encoded_parity, *encoded_parity, m);
485 goto out;
486 }
487
488 ret = finalize_fragments_after_encode(instance, k, m, blocksize, orig_data_size,
489 *encoded_data, *encoded_parity);
490
491 *fragment_len = get_fragment_size((*encoded_data)[0]);
492
493out:
494 if (ret) {
495 /* Cleanup the allocations we have done */
496 liberasurecode_encode_cleanup(desc, *encoded_data, *encoded_parity);
497 log_error("Error in liberasurecode_encode %d", ret);
498 }
499 return ret;
500}
501
515int liberasurecode_decode_cleanup(int desc, char *data)
516{
517 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
518 if (NULL == instance) {
519 return -EBACKENDNOTAVAIL;
520 }
521
522 free(data);
523
524 return 0;
525}
526
541 char **available_fragments, /* input */
542 int num_fragments, uint64_t fragment_len, /* input */
543 int force_metadata_checks, /* input */
544 char **out_data, uint64_t *out_data_len) /* output */
545{
546 int i, j;
547 int ret = 0;
548
549 int k = -1, m = -1;
550 int orig_data_size = 0;
551
552 int blocksize = 0;
553 char **data = NULL;
554 char **parity = NULL;
555 char **data_segments = NULL;
556 char **parity_segments = NULL;
557 int *missing_idxs = NULL;
558
559 uint64_t realloc_bm = 0;
560
561 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
562 if (NULL == instance) {
563 ret = -EBACKENDNOTAVAIL;
564 goto out;
565 }
566
567 if (NULL == available_fragments) {
568 log_error("Pointer to encoded fragments buffer is null!");
569 ret = -EINVALIDPARAMS;
570 goto out;
571 }
572
573 if (NULL == out_data) {
574 log_error("Pointer to decoded data buffer is null!");
575 ret = -EINVALIDPARAMS;
576 goto out;
577 }
578
579 if (NULL == out_data_len) {
580 log_error("Pointer to decoded data length variable is null!");
581 ret = -EINVALIDPARAMS;
582 goto out;
583 }
584
585 k = instance->args.uargs.k;
586 m = instance->args.uargs.m;
587
588 if (num_fragments < k) {
589 log_error("Not enough fragments to decode, got %d, need %d!",
590 num_fragments, k);
591 ret = -EINSUFFFRAGS;
592 goto out;
593 }
594
595 if (fragment_len < sizeof(fragment_header_t)) {
596 log_error("Fragments not long enough to include headers! "
597 "Need %zu, but got %lu.", sizeof(fragment_header_t),
598 (unsigned long)fragment_len);
599 ret = -EBADHEADER;
600 goto out;
601 }
602 for (i = 0; i < num_fragments; ++i) {
603 /* Verify metadata checksum */
605 (fragment_header_t *) available_fragments[i])) {
606 log_error("Invalid fragment header information!");
607 ret = -EBADHEADER;
608 goto out;
609 }
610 }
611
612 if (instance->common.id != EC_BACKEND_SHSS && instance->common.id != EC_BACKEND_LIBPHAZR) {
613 /* shss (ntt_backend) & libphazr backend must force to decode */
614 // TODO: Add a frag and function to handle whether the backend want to decode or not.
615 /*
616 * Try to re-assebmle the original data before attempting a decode
617 */
618 ret = fragments_to_string(k, m,
619 available_fragments, num_fragments,
620 out_data, out_data_len);
621
622 if (ret == 0) {
623 /* We were able to get the original data without decoding! */
624 goto out;
625 }
626 }
627
628 /*
629 * Allocate arrays for data, parity and missing_idxs
630 */
631 data = alloc_zeroed_buffer(sizeof(char*) * k);
632 if (NULL == data) {
633 log_error("Could not allocate data buffer!");
634 goto out;
635 }
636
637 parity = alloc_zeroed_buffer(sizeof(char*) * m);
638 if (NULL == parity) {
639 log_error("Could not allocate parity buffer!");
640 goto out;
641 }
642
643 missing_idxs = alloc_and_set_buffer(sizeof(char*) * (k + m), -1);
644 if (NULL == missing_idxs) {
645 log_error("Could not allocate missing_idxs buffer!");
646 goto out;
647 }
648
649 /* If metadata checks requested, check fragment integrity upfront */
650 if (force_metadata_checks) {
651 int num_invalid_fragments = 0;
652 for (i = 0; i < num_fragments; ++i) {
653 if (is_invalid_fragment(desc, available_fragments[i])) {
654 ++num_invalid_fragments;
655 }
656 }
657 if ((num_fragments - num_invalid_fragments) < k) {
658 ret = -EINSUFFFRAGS;
659 log_error("Not enough valid fragments available for decode!");
660 goto out;
661 }
662 }
663
664 /*
665 * Separate the fragments into data and parity. Also determine which
666 * pieces are missing.
667 */
668 ret = get_fragment_partition(k, m, available_fragments, num_fragments,
669 data, parity, missing_idxs);
670
671 if (ret < 0) {
672 log_error("Could not properly partition the fragments!");
673 goto out;
674 }
675
676 /*
677 * Preparing the fragments for decode. This will alloc aligned buffers
678 * when unaligned buffers were passed in available_fragments. It passes
679 * back a bitmap telling us which buffers need to be freed by us
680 * (realloc_bm).
681 *
682 */
684 data, parity, missing_idxs,
685 &orig_data_size, &blocksize,
686 fragment_len, &realloc_bm);
687 if (ret < 0) {
688 log_error("Could not prepare fragments for decode!");
689 goto out;
690 }
691
692 data_segments = alloc_zeroed_buffer(k * sizeof(char *));
693 parity_segments = alloc_zeroed_buffer(m * sizeof(char *));
694 get_data_ptr_array_from_fragments(data_segments, data, k);
695 get_data_ptr_array_from_fragments(parity_segments, parity, m);
696
697 /* call the backend decode function passing it desc instance */
698 ret = instance->common.ops->decode(instance->desc.backend_desc,
699 data_segments, parity_segments,
700 missing_idxs, blocksize);
701
702 if (ret < 0) {
703 log_error("Encountered error in backend decode function!");
704 goto out;
705 }
706
707 /*
708 * Need to fill in the missing data headers so we can generate
709 * the original string.
710 */
711 j = 0;
712 while (missing_idxs[j] >= 0) {
713 int set_chksum = 1;
714 int missing_idx = missing_idxs[j];
715 if (missing_idx < k) {
716 /* Generate headers */
717 char *fragment_ptr = data[missing_idx];
718 init_fragment_header(fragment_ptr);
719 add_fragment_metadata(instance, fragment_ptr, missing_idx,
720 orig_data_size, blocksize, instance->args.uargs.ct,
721 !set_chksum);
722 }
723 j++;
724 }
725
726 /* Try to generate the original string */
727 ret = fragments_to_string(k, m, data, k, out_data, out_data_len);
728
729 if (ret < 0) {
730 log_error("Could not convert decoded fragments to a string!");
731 }
732
733out:
734 /* Free the buffers allocated in prepare_fragments_for_decode */
735 if (realloc_bm != 0) {
736 for (i = 0; i < k; i++) {
737 if (realloc_bm & (1 << i)) {
738 free(data[i]);
739 }
740 }
741
742 for (i = 0; i < m; i++) {
743 if (realloc_bm & (1 << (i + k))) {
744 free(parity[i]);
745 }
746 }
747 }
748
749 free(data);
750 free(parity);
751 free(missing_idxs);
752 free(data_segments);
753 free(parity_segments);
754
755 return ret;
756}
757
771 char **available_fragments, /* input */
772 int num_fragments, uint64_t fragment_len, /* input */
773 int destination_idx, /* input */
774 char* out_fragment) /* output */
775{
776 int ret = 0;
777 int blocksize = 0;
778 int orig_data_size = 0;
779 char **data = NULL;
780 char **parity = NULL;
781 int *missing_idxs = NULL;
782 char *fragment_ptr = NULL;
783 int is_destination_missing = 0;
784 int k = -1;
785 int m = -1;
786 int i;
787 uint64_t realloc_bm = 0;
788 char **data_segments = NULL;
789 char **parity_segments = NULL;
790 int set_chksum = 1;
791
792 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
793 if (NULL == instance) {
794 ret = -EBACKENDNOTAVAIL;
795 goto out;
796 }
797
798 if (NULL == available_fragments) {
799 log_error("Can not reconstruct fragment, available fragments pointer is NULL");
800 ret = -EINVALIDPARAMS;
801 goto out;
802 }
803
804 if (NULL == out_fragment) {
805 log_error("Can not reconstruct fragment, output fragment pointer is NULL");
806 ret = -EINVALIDPARAMS;
807 goto out;
808 }
809
810 k = instance->args.uargs.k;
811 m = instance->args.uargs.m;
812
813 for (i = 0; i < num_fragments; i++) {
814 /* Verify metadata checksum */
816 (fragment_header_t *) available_fragments[i])) {
817 log_error("Invalid fragment header information!");
818 ret = -EBADHEADER;
819 goto out;
820 }
821 }
822
823 /*
824 * Allocate arrays for data, parity and missing_idxs
825 */
826 data = alloc_zeroed_buffer(sizeof(char*) * k);
827 if (NULL == data) {
828 log_error("Could not allocate data buffer!");
829 ret = -ENOMEM;
830 goto out;
831 }
832
833 parity = alloc_zeroed_buffer(sizeof(char*) * m);
834 if (NULL == parity) {
835 log_error("Could not allocate parity buffer!");
836 ret = -ENOMEM;
837 goto out;
838 }
839
840 missing_idxs = alloc_and_set_buffer(sizeof(int*) * (k + m), -1);
841 if (NULL == missing_idxs) {
842 log_error("Could not allocate missing_idxs buffer!");
843 ret = -ENOMEM;
844 goto out;
845 }
846
847 /*
848 * Separate the fragments into data and parity. Also determine which
849 * pieces are missing.
850 */
851 ret = get_fragment_partition(k, m, available_fragments, num_fragments,
852 data, parity, missing_idxs);
853
854 if (ret < 0) {
855 log_error("Could not properly partition the fragments!");
856 goto out;
857 }
858
859 /*
860 * Odd corner-case: If the caller passes in a destination_idx that
861 * is also included in the available fragments list, we should *not*
862 * try to reconstruct.
863 *
864 * For now, we will log a warning and do nothing. In the future, we
865 * should probably log and return an error.
866 *
867 */
868 i = 0;
869 while (missing_idxs[i] > -1) {
870 if (missing_idxs[i] == destination_idx) {
871 is_destination_missing = 1;
872 }
873 i++;
874 }
875
876 if (!is_destination_missing) {
877 if (destination_idx < k) {
878 fragment_ptr = data[destination_idx];
879 } else {
880 fragment_ptr = parity[destination_idx - k];
881 }
882 log_warn("Dest idx for reconstruction was supplied as available buffer!");
883 goto destination_available;
884 }
885
886 /*
887 * Preparing the fragments for reconstruction. This will alloc aligned
888 * buffers when unaligned buffers were passed in available_fragments.
889 * It passes back a bitmap telling us which buffers need to be freed by
890 * us (realloc_bm).
891 */
892 ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
893 &orig_data_size, &blocksize,
894 fragment_len, &realloc_bm);
895 if (ret < 0) {
896 log_error("Could not prepare fragments for reconstruction!");
897 goto out;
898 }
899 data_segments = alloc_zeroed_buffer(k * sizeof(char *));
900 parity_segments = alloc_zeroed_buffer(m * sizeof(char *));
901 get_data_ptr_array_from_fragments(data_segments, data, k);
902 get_data_ptr_array_from_fragments(parity_segments, parity, m);
903
904
905 /* call the backend reconstruct function passing it desc instance */
906 ret = instance->common.ops->reconstruct(instance->desc.backend_desc,
907 data_segments, parity_segments,
908 missing_idxs, destination_idx,
909 blocksize);
910 if (ret < 0) {
911 log_error("Could not reconstruct fragment!");
912 goto out;
913 }
914
915 /*
916 * Update the header to reflect the newly constructed fragment
917 */
918 if (destination_idx < k) {
919 fragment_ptr = data[destination_idx];
920 } else {
921 fragment_ptr = parity[destination_idx - k];
922 }
923 init_fragment_header(fragment_ptr);
924 add_fragment_metadata(instance, fragment_ptr, destination_idx,
925 orig_data_size, blocksize, instance->args.uargs.ct,
926 set_chksum);
927
928destination_available:
929 /*
930 * Copy the reconstructed fragment to the output buffer
931 *
932 * Note: the address stored in fragment_ptr will be freed below
933 */
934 memcpy(out_fragment, fragment_ptr, fragment_len);
935
936out:
937 /* Free the buffers allocated in prepare_fragments_for_decode */
938 if (realloc_bm != 0) {
939 for (i = 0; i < k; i++) {
940 if (realloc_bm & (1 << i)) {
941 free(data[i]);
942 }
943 }
944
945 for (i = 0; i < m; i++) {
946 if (realloc_bm & (1 << (i + k))) {
947 free(parity[i]);
948 }
949 }
950 }
951
952 free(data);
953 free(parity);
954 free(missing_idxs);
955 free(data_segments);
956 free(parity_segments);
957
958 return ret;
959}
960
977 int *fragments_to_reconstruct,
978 int *fragments_to_exclude,
979 int *fragments_needed)
980{
981 int ret = 0;
982
983 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
984 if (NULL == instance) {
985 ret = -EBACKENDNOTAVAIL;
986 goto out_error;
987 }
988 if (NULL == fragments_to_reconstruct) {
989 log_error("Unable to determine list of fragments needed, pointer to list of indexes to reconstruct is NULL.");
990 ret = -EINVALIDPARAMS;
991 goto out_error;
992 }
993
994 if (NULL == fragments_to_exclude) {
995 log_error("Unable to determine list of fragments needed, pointer to list of fragments to exclude is NULL.");
996 ret = -EINVALIDPARAMS;
997 goto out_error;
998 }
999
1000 if (NULL == fragments_needed) {
1001 log_error("Unable to determine list of fragments needed, pointer to list of fragments to reconstruct is NULL.");
1002 ret = -EINVALIDPARAMS;
1003 goto out_error;
1004 }
1005
1006 /* FIXME preprocessing */
1007
1008 /* call the backend fragments_needed function passing it desc instance */
1009 ret = instance->common.ops->fragments_needed(
1010 instance->desc.backend_desc,
1011 fragments_to_reconstruct, fragments_to_exclude, fragments_needed);
1012
1013out_error:
1014 return ret;
1015}
1016
1017/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
1018
1030 fragment_metadata_t *fragment_metadata)
1031{
1032 int ret = 0;
1033 fragment_header_t *fragment_hdr = NULL;
1034
1035 if (NULL == fragment) {
1036 log_error("Need valid fragment object to get metadata for");
1037 ret = -EINVALIDPARAMS;
1038 goto out;
1039 }
1040
1041 if (NULL == fragment_metadata) {
1042 log_error("Need valid fragment_metadata object for return value");
1043 ret = -EINVALIDPARAMS;
1044 goto out;
1045 }
1046
1047 /* Verify metadata checksum */
1048 if (is_invalid_fragment_header((fragment_header_t *) fragment)) {
1049 log_error("Invalid fragment header information!");
1050 ret = -EBADHEADER;
1051 goto out;
1052 }
1053
1054 memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
1055 fragment_hdr = (fragment_header_t *) fragment;
1056 if (LIBERASURECODE_FRAG_HEADER_MAGIC != fragment_hdr->magic) {
1057 if (LIBERASURECODE_FRAG_HEADER_MAGIC != bswap_32(fragment_hdr->magic)) {
1058 log_error("Invalid fragment, illegal magic value");
1059 ret = -EINVALIDPARAMS;
1060 goto out;
1061 } else {
1062 // Must've written this on an opposite-endian architecture.
1063 // Fix it in fragment_metadata
1064 fragment_metadata->idx = bswap_32(fragment_metadata->idx);
1065 fragment_metadata->size = bswap_32(fragment_metadata->size);
1066 fragment_metadata->frag_backend_metadata_size =
1067 bswap_32(fragment_metadata->frag_backend_metadata_size);
1068 fragment_metadata->orig_data_size =
1069 bswap_64(fragment_metadata->orig_data_size);
1070 fragment_metadata->chksum_type =
1071 bswap_32(fragment_metadata->chksum_type);
1072 for (int i = 0; i < LIBERASURECODE_MAX_CHECKSUM_LEN; i++) {
1073 fragment_metadata->chksum[i] =
1074 bswap_32(fragment_metadata->chksum[i]);
1075 }
1076 fragment_metadata->backend_version =
1077 bswap_32(fragment_metadata->backend_version);
1078 }
1079 }
1080
1081 switch(fragment_metadata->chksum_type) {
1082 case CHKSUM_CRC32: {
1083 uint32_t computed_chksum = 0;
1084 uint32_t stored_chksum = fragment_metadata->chksum[0];
1085 char *fragment_data = get_data_ptr_from_fragment(fragment);
1086 uint64_t fragment_size = fragment_metadata->size;
1087 computed_chksum = crc32(0, (unsigned char *) fragment_data, fragment_size);
1088 if (stored_chksum != computed_chksum) {
1089 // Try again with our "alternative" crc32; see
1090 // https://bugs.launchpad.net/liberasurecode/+bug/1666320
1091 computed_chksum = liberasurecode_crc32_alt(
1092 0, fragment_data, fragment_size);
1093 if (stored_chksum != computed_chksum) {
1094 fragment_metadata->chksum_mismatch = 1;
1095 } else {
1096 fragment_metadata->chksum_mismatch = 0;
1097 }
1098 } else {
1099 fragment_metadata->chksum_mismatch = 0;
1100 }
1101 break;
1102 }
1103 case CHKSUM_MD5:
1104 break;
1105 case CHKSUM_NONE:
1106 default:
1107 break;
1108 }
1109
1110out:
1111 return ret;
1112}
1113
1114int is_invalid_fragment_header(fragment_header_t *header)
1115{
1116 uint32_t csum = 0, metadata_chksum = 0, libec_version = 0;
1117 assert (NULL != header);
1118 if (header->libec_version == 0)
1119 /* libec_version must be bigger than 0 */
1120 return 1;
1121 metadata_chksum = header->metadata_chksum;
1122 libec_version = header->libec_version;
1123 if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
1124 if (bswap_32(header->magic) != LIBERASURECODE_FRAG_HEADER_MAGIC) {
1125 log_error("Invalid fragment header (get meta chksum)!");
1126 return 1;
1127 } else {
1128 // Must've written this on an opposite-endian architecture.
1129 // Fix our reference checksum and version, but *don't touch
1130 // the header data yet*. Leave that for when we're extracting
1131 // in liberasurecode_get_fragment_metadata
1132 metadata_chksum = bswap_32(metadata_chksum);
1133 libec_version = bswap_32(libec_version);
1134 }
1135 }
1136
1137 if (libec_version < _VERSION(1,2,0))
1138 /* no metadata checksum support */
1139 return 0;
1140
1141 csum = crc32(0, (unsigned char *) &header->meta, sizeof(fragment_metadata_t));
1142 if (metadata_chksum == csum) {
1143 return 0;
1144 }
1145 // Else, try again with our "alternative" crc32; see
1146 // https://bugs.launchpad.net/liberasurecode/+bug/1666320
1147 csum = liberasurecode_crc32_alt(0, &header->meta, sizeof(fragment_metadata_t));
1148 return (metadata_chksum != csum);
1149}
1150
1152 fragment_metadata_t *md)
1153{
1154 int k = be->args.uargs.k;
1155 int m = be->args.uargs.m;
1156 if (md->idx > (k + m)) {
1157 return 1;
1158 }
1159 if (md->backend_id != be->common.id) {
1160 return 1;
1161 }
1162 if (!be->common.ops->is_compatible_with(md->backend_version)) {
1163 return 1;
1164 }
1165 return 0;
1166}
1167
1168int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
1169{
1170 ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
1171 if (!be) {
1172 log_error("Unable to verify fragment metadata: invalid backend id %d.",
1173 desc);
1174 return -EINVALIDPARAMS;
1175 }
1177 fragment_metadata) != 0) {
1178 return -EBADHEADER;
1179 }
1180 if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
1181 return -EBADHEADER;
1182 }
1183 if (fragment_metadata->chksum_mismatch == 1) {
1184 return -EBADCHKSUM;
1185 }
1186 return 0;
1187}
1188
1189int is_invalid_fragment(int desc, char *fragment)
1190{
1191 uint32_t ver = 0;
1192 fragment_metadata_t fragment_metadata;
1193 ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
1194 if (!be) {
1195 log_error("Unable to verify fragment metadata: invalid backend id %d.",
1196 desc);
1197 return 1;
1198 }
1199 if (!fragment) {
1200 log_error("Unable to verify fragment validity: fragments missing.");
1201 return 1;
1202 }
1203 if (get_libec_version(fragment, &ver) != 0 ||
1204 ver > LIBERASURECODE_VERSION) {
1205 return 1;
1206 }
1207 if (liberasurecode_get_fragment_metadata(fragment, &fragment_metadata) != 0) {
1208 return 1;
1209 }
1210 if (is_invalid_fragment_metadata(desc, &fragment_metadata) != 0) {
1211 return 1;
1212 }
1213 return 0;
1214}
1215
1217 char **fragments, int num_fragments)
1218{
1219 int i = 0;
1220 if (!fragments) {
1221 log_error("Unable to verify stripe metadata: fragments missing.");
1222 return -EINVALIDPARAMS;
1223 }
1224 if (num_fragments <= 0) {
1225 log_error("Unable to verify stripe metadata: "
1226 "number of fragments must be greater than 0.");
1227 return -EINVALIDPARAMS;
1228 }
1229
1230 for (i = 0; i < num_fragments; i++) {
1231 fragment_metadata_t *fragment_metadata = (fragment_metadata_t*)fragments[i];
1232 int ret = is_invalid_fragment_metadata(desc, fragment_metadata);
1233 if (ret < 0) {
1234 return ret;
1235 }
1236 }
1237
1238 return 0;
1239}
1240
1241/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
1242
1250int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len)
1251{
1252 int k;
1253 int ret = 0;
1254 int word_size;
1255 int alignment_multiple;
1256
1257 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
1258 if (NULL == instance) {
1259 ret = -EBACKENDNOTAVAIL;
1260 goto out;
1261 }
1262
1263 k = instance->args.uargs.k;
1264
1265 word_size = instance->common.ops->element_size(
1266 instance->desc.backend_desc) / 8;
1267
1268 alignment_multiple = k * word_size;
1269
1270 ret = ((data_len + alignment_multiple - 1) / alignment_multiple)
1271 * alignment_multiple;
1272
1273out:
1274 return ret;
1275}
1276
1282{
1284}
1285
1286int liberasurecode_get_fragment_size(int desc, int data_len)
1287{
1288 ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc);
1289 // TODO: Create a common function to calculate fragment size also for preprocessing
1290 if (NULL == instance)
1291 return -EBACKENDNOTAVAIL;
1292 int aligned_data_len = get_aligned_data_size(instance, data_len);
1293 int blocksize = aligned_data_len / instance->args.uargs.k;
1294 int metadata_size = instance->common.ops->get_backend_metadata_size(
1295 instance->desc.backend_desc,
1296 blocksize);
1297 int size = blocksize + metadata_size;
1298
1299 return size;
1300}
1301
1302
1308{
1309 return LIBERASURECODE_VERSION;
1310}
1311
1312/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=* misc *=~==~=*=~==~=*=~==~=*=~==~=*=~== */
1313
1314#if 0
1315/* Validate backend before calling init */
1316int liberasurecode_backend_validate(ec_backend_t backend)
1317{
1318 /* Verify that the backend implements all required methods */
1319}
1320
1321/* FIXME - do we need to use reference counts if we are creating
1322* a new instance per user */
1323
1324/* Get a reference to an EC backend */
1325ec_backend_t liberasurecode_backend_get(const char *name)
1326{
1327 ec_backend_t b = liberasurecode_backend_lookup_by_name(name);
1328 if (NULL != b)
1329 ++b->users;
1330 return b;
1331}
1332
1333/* Drop an EC backend reference held */
1334void liberasurecode_backend_put(ec_backend_t backend)
1335{
1336 if (backend->users > 0)
1337 --backend->users;
1338}
1339
1340/* Query interface for active instances */
1341ec_backend_t liberasurecode_backend_instance_active(ec_backend_t instance)
1342{
1343 ec_backend_t b;
1344
1345 SLIST_FOREACH(b, &active_instances, link) {
1346 if (strcmp(b->name, name) == 0)
1347 return b;
1348 }
1349
1350 return NULL;
1351}
1352
1353ec_backend_t liberasurecode_backend_lookup_by_soname(const char *soname)
1354{
1355 ec_backend_t b;
1356
1357 SLIST_FOREACH(b, &active_instances, link) {
1358 if (strcmp(b->soname, soname) == 0)
1359 return b;
1360 }
1361
1362 return NULL;
1363}
1364#endif
1365
1366/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
int liberasurecode_crc32_alt(int crc, const void *buf, size_t size)
Definition crc32.c:92
int liberasurecode_verify_fragment_metadata(ec_backend_t be, fragment_metadata_t *md)
int liberasurecode_decode(int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int force_metadata_checks, char **out_data, uint64_t *out_data_len)
Reconstruct original data from a set of k encoded fragments.
int liberasurecode_encode(int desc, const char *orig_data, uint64_t orig_data_size, char ***encoded_data, char ***encoded_parity, uint64_t *fragment_len)
Erasure encode a data buffer.
void * liberasurecode_backend_open(ec_backend_t instance)
int is_invalid_fragment_header(fragment_header_t *header)
struct ec_backend_common backend_flat_xor_hd
Definition flat_xor_hd.c:51
int liberasurecode_backend_close(ec_backend_t instance)
static void print_dlerror(const char *caller)
int liberasurecode_decode_cleanup(int desc, char *data)
Cleanup structures allocated by librasurecode_decode.
int liberasurecode_fragments_needed(int desc, int *fragments_to_reconstruct, int *fragments_to_exclude, int *fragments_needed)
Return a list of lists with valid rebuild indexes given a list of missing indexes.
struct ec_backend_common backend_null
Definition null.c:231
struct ec_backend_common backend_shss
Definition shss.c:43
struct ec_backend_common backend_isa_l_rs_vand
int liberasurecode_instance_create(const ec_backend_id_t id, struct ec_args *args)
Create a liberasurecode instance and return a descriptor for use with EC operations (encode,...
struct ec_backend_common backend_liberasurecode_rs_vand
struct ec_backend_common backend_isa_l_rs_cauchy
uint32_t liberasurecode_get_version()
This will return the liberasurecode version for the descriptor.
ec_backend_t ec_backends_supported[]
Definition erasurecode.c:56
int liberasurecode_get_fragment_metadata(char *fragment, fragment_metadata_t *fragment_metadata)
Get opaque metadata for a fragment.
int liberasurecode_get_minimum_encode_size(int desc)
This will return the minumum encode size, which is the minimum buffer size that can be encoded.
int liberasurecode_get_aligned_data_size(int desc, uint64_t data_len)
This computes the aligned size of a buffer passed into the encode function.
SLIST_HEAD(backend_list, ec_backend)
Look up a backend instance by descriptor.
Definition erasurecode.c:76
int is_invalid_fragment(int desc, char *fragment)
int liberasurecode_reconstruct_fragment(int desc, char **available_fragments, int num_fragments, uint64_t fragment_len, int destination_idx, char *out_fragment)
Reconstruct a missing fragment from a subset of available fragments.
char * ec_backends_supported_str[EC_BACKENDS_MAX]
Definition erasurecode.c:71
int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
int liberasurecode_verify_stripe_metadata(int desc, char **fragments, int num_fragments)
struct ec_backend_common backend_libphazr
Definition libphazr.c:50
int liberasurecode_backend_instance_unregister(ec_backend_t instance)
Unregister a backend instance.
int liberasurecode_encode_cleanup(int desc, char **encoded_data, char **encoded_parity)
Cleanup structures allocated by librasurecode_encode.
int num_supported_backends
Definition erasurecode.c:70
int liberasurecode_backend_available(const ec_backend_id_t backend_id)
Checks if a given backend is available.
int liberasurecode_backend_alloc_desc(void)
Allocated backend instance descriptor.
struct ec_backend_common backend_jerasure_rs_cauchy
int liberasurecode_backend_instance_register(ec_backend_t instance)
Register a backend instance with liberasurecode.
int liberasurecode_instance_destroy(int desc)
Close a liberasurecode instance.
struct ec_backend_common backend_jerasure_rs_vand
int liberasurecode_get_fragment_size(int desc, int data_len)
void __attribute__((constructor))
void * alloc_zeroed_buffer(int size)
Allocate a zero-ed buffer of a specific size.
void * alloc_and_set_buffer(int size, int value)
Allocate a buffer of a specific size and set its' contents to the specified value.
int get_aligned_data_size(ec_backend_t instance, int data_len)
Compute a size aligned to the number of data and the underlying wordsize of the EC algorithm.
int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments)
uint64_t get_fragment_size(char *buf)
Return total fragment length (on-disk, on-wire)
int get_fragment_ptr_array_from_data(char **frag_array, char **data, int num_data)
char * get_data_ptr_from_fragment(char *buf)
int get_libec_version(char *buf, uint32_t *ver)
void add_fragment_metadata(ec_backend_t be, char *fragment, int idx, uint64_t orig_data_size, int blocksize, ec_checksum_type_t ct, int add_chksum)
int finalize_fragments_after_encode(ec_backend_t instance, int k, int m, int blocksize, uint64_t orig_data_size, char **encoded_data, char **encoded_parity)
int prepare_fragments_for_encode(ec_backend_t instance, int k, int m, const char *orig_data, uint64_t orig_data_size, char **encoded_data, char **encoded_parity, int *blocksize)
int prepare_fragments_for_decode(int k, int m, char **data, char **parity, int *missing_idxs, int *orig_size, int *fragment_payload_size, int fragment_size, uint64_t *realloc_bm)
int fragments_to_string(int k, int m, char **fragments, int num_fragments, char **orig_payload, uint64_t *payload_len)
int get_fragment_partition(int k, int m, char **fragments, int num_fragments, char **data, char **parity, int *missing)