213 MPI_Topo_test(comm, &comm_type);
214 if (comm_type == MPI_CART) {
216 balancer->setCommunicator(communicator);
218 const int dimension = balancer->getDimension();
219 int location[dimension];
221 int periodicity[dimension];
222 MPI_Cart_get(communicator, dimension, size, periodicity, location);
223 procGridLoc.assign(location, location + dimension);
224 procGridDim.assign(size, size + dimension);
228 MPI_Comm_rank(comm, &rank);
233 switch (balancer->getDimension()) {
235 idx = procGridLoc.at(2) + procGridLoc.at(1) * procGridDim.at(2) +
236 procGridLoc.at(0) * procGridDim.at(2) * procGridDim.at(1);
239 idx = procGridLoc.at(1) + procGridLoc.at(0) * procGridDim.at(1);
242 idx = procGridLoc.at(0);
246 __FILE__, __func__, __LINE__,
247 "ALL cannot deal with more then three dimensions (or less then "
254 MPI_Comm_split(comm, 0, idx, &temp_comm);
256 std::vector<int> periods(3, 1);
259 MPI_Cart_create(temp_comm, balancer->getDimension(), procGridDim.data(),
260 periods.data(), 0, &communicator);
261 balancer->setCommunicator(communicator);
264 __FILE__, __func__, __LINE__,
265 "When using a non-cartesian communicator process grid parameters "
266 "required (not set).");
521 auto points = vtkSmartPointer<vtkPoints>::New();
523 points->Allocate(8 * balancer->getDimension());
529 controller = vtkMPIController::New();
530 controller->Initialize();
531 controller->SetGlobalController(controller);
535 MPI_Comm_rank(communicator, &local_rank);
536 MPI_Comm_size(communicator, &n_ranks);
538 std::vector<Point<W>> tmp_outline(2);
539 std::vector<W> tmp_0(
540 {outline.at(0)[0], outline.at(0)[1], outline.at(0)[2]});
541 std::vector<W> tmp_1(
542 {outline.at(1)[0], outline.at(1)[1], outline.at(1)[2]});
543 tmp_outline.at(0) =
Point<W>(tmp_0);
544 tmp_outline.at(1) =
Point<W>(tmp_1);
546 for (
auto z = 0; z <= 1; ++z)
547 for (
auto y = 0; y <= 1; ++y)
548 for (
auto x = 0; x <= 1; ++x) {
549 points->InsertPoint(x + 2 * y + 4 * z, tmp_outline.at(x)[0],
550 tmp_outline.at(y)[1], tmp_outline.at(z)[2]);
553 auto hexa = vtkSmartPointer<vtkVoxel>::New();
554 for (
int i = 0; i < 8; ++i)
555 hexa->GetPointIds()->SetId(i, i);
557 auto cellArray = vtkSmartPointer<vtkCellArray>::New();
558 cellArray->InsertNextCell(hexa);
561 auto work = vtkSmartPointer<vtkFloatArray>::New();
562 work->SetNumberOfComponents(1);
563 work->SetNumberOfTuples(1);
564 work->SetName(
"work");
565 W total_work = std::accumulate(balancer->getWork().begin(),
566 balancer->getWork().end(), (W)0);
567 work->SetValue(0, total_work);
571 MPI_Comm_rank(communicator, &rank);
572 auto coords = vtkSmartPointer<vtkFloatArray>::New();
573 coords->SetNumberOfComponents(3);
574 coords->SetNumberOfTuples(1);
575 coords->SetName(
"coords");
576 coords->SetValue(0, procGridLoc.at(0));
577 coords->SetValue(1, procGridLoc.at(1));
578 coords->SetValue(2, procGridLoc.at(2));
580 auto rnk = vtkSmartPointer<vtkFloatArray>::New();
581 rnk->SetNumberOfComponents(1);
582 rnk->SetNumberOfTuples(1);
583 rnk->SetName(
"MPI rank");
584 rnk->SetValue(0, rank);
587 auto tag = vtkSmartPointer<vtkIntArray>::New();
588 tag->SetNumberOfComponents(1);
589 tag->SetNumberOfTuples(1);
591 tag->SetValue(0, procTag);
602 for (
int i = 0; i < 3; ++i) {
603 local_min[i] = outline.at(0)[i];
604 local_max[i] = outline.at(1)[i];
605 width[i] = local_max[i] - local_min[i];
609 W surface = (W)2.0 * width[0] * width[1] + (W)2.0 * width[1] * width[2] +
610 (W)2.0 * width[0] * width[2];
612 auto expanse = vtkSmartPointer<vtkFloatArray>::New();
613 expanse->SetNumberOfComponents(6);
614 expanse->SetNumberOfTuples(1);
615 expanse->SetName(
"expanse");
616 expanse->SetValue(0, width[0]);
617 expanse->SetValue(1, width[0]);
618 expanse->SetValue(2, width[0]);
619 expanse->SetValue(3, volume);
620 expanse->SetValue(4, surface);
621 expanse->SetValue(5, surface / volume);
623 MPI_Allreduce(&local_min, &global_min, 3, MPIDataTypeW, MPI_MIN,
625 MPI_Allreduce(&local_max, &global_max, 3, MPIDataTypeW, MPI_MAX,
628 for (
int i = 0; i < 3; ++i) {
629 global_extent[2 * i] = global_min[i];
630 global_extent[2 * i + 1] = global_max[i];
634 auto unstructuredGrid = vtkSmartPointer<vtkUnstructuredGrid>::New();
635 unstructuredGrid->SetPoints(points);
636 unstructuredGrid->SetCells(VTK_VOXEL, cellArray);
637 unstructuredGrid->GetCellData()->AddArray(work);
638 unstructuredGrid->GetCellData()->AddArray(expanse);
639 unstructuredGrid->GetCellData()->AddArray(rnk);
640 unstructuredGrid->GetCellData()->AddArray(coords);
641 unstructuredGrid->GetCellData()->AddArray(tag);
643 createDirectory(
"vtk_outline");
644 std::ostringstream ss_local;
645 ss_local <<
"vtk_outline/ALL_vtk_outline_" << std::setw(7)
646 << std::setfill(
'0') << step <<
"_" << local_rank <<
".vtu";
648 auto writer = vtkSmartPointer<vtkXMLUnstructuredGridWriter>::New();
649 writer->SetInputData(unstructuredGrid);
650 writer->SetFileName(ss_local.str().c_str());
651 writer->SetDataModeToAscii();
657 std::ostringstream ss_para;
658 ss_para <<
"vtk_outline/ALL_vtk_outline_" << std::setw(7)
659 << std::setfill(
'0') << step <<
".pvtu";
661 auto parallel_writer =
662 vtkSmartPointer<vtkXMLPUnstructuredGridWriter>::New();
663 parallel_writer->SetFileName(ss_para.str().c_str());
664 parallel_writer->SetNumberOfPieces(n_ranks);
665 parallel_writer->SetStartPiece(local_rank);
666 parallel_writer->SetEndPiece(local_rank);
667 parallel_writer->SetInputData(unstructuredGrid);
668 parallel_writer->SetDataModeToAscii();
670 parallel_writer->Write();
682 MPI_Comm_rank(communicator, &local_rank);
683 MPI_Comm_size(communicator, &n_ranks);
687 T local_vertices[nVertices * balancer->getDimension() + 1];
689 for (
int v = 0; v < nVertices; ++v) {
690 for (
int d = 0; d < balancer->getDimension(); ++d) {
691 local_vertices[v * balancer->getDimension() + d] =
692 balancer->getVertices().at(v)[d];
695 local_vertices[nVertices * balancer->getDimension()] =
696 (T)balancer->getWork().at(0);
705 T global_vertices[n_ranks * (nVertices * balancer->getDimension() + 1)];
708 MPI_Gather(local_vertices, nVertices * balancer->getDimension() + 1,
709 MPIDataTypeT, global_vertices,
710 nVertices * balancer->getDimension() + 1, MPIDataTypeT, 0,
713 if (local_rank == 0) {
714 auto vtkpoints = vtkSmartPointer<vtkPoints>::New();
715#ifdef VTK_CELL_ARRAY_V2
716 vtkNew<vtkUnstructuredGrid> unstructuredGrid;
717 unstructuredGrid->Allocate(n_ranks + 10);
719 auto unstructuredGrid = vtkSmartPointer<vtkUnstructuredGrid>::New();
723 for (
int i = 0; i < n_ranks; ++i) {
724 for (
int v = 0; v < nVertices; ++v) {
725 vtkpoints->InsertNextPoint(
726 global_vertices[i * (nVertices * balancer->getDimension() + 1) +
727 v * balancer->getDimension() + 0],
728 global_vertices[i * (nVertices * balancer->getDimension() + 1) +
729 v * balancer->getDimension() + 1],
730 global_vertices[i * (nVertices * balancer->getDimension() + 1) +
731 v * balancer->getDimension() + 2]);
734 unstructuredGrid->SetPoints(vtkpoints);
737 auto work = vtkSmartPointer<vtkFloatArray>::New();
738 auto cell = vtkSmartPointer<vtkFloatArray>::New();
739 work->SetNumberOfComponents(1);
740 work->SetNumberOfTuples(n_ranks);
741 work->SetName(
"work");
742 cell->SetNumberOfComponents(1);
743 cell->SetNumberOfTuples(n_ranks);
744 cell->SetName(
"cell id");
746 for (
int n = 0; n < n_ranks; ++n) {
748#ifdef VTK_CELL_ARRAY_V2
750 vtkIdType pointIds[8] = {8 * n + 0, 8 * n + 1, 8 * n + 2, 8 * n + 3,
751 8 * n + 4, 8 * n + 5, 8 * n + 6, 8 * n + 7};
753 vtkIdType faces[48] = {
754 3, 8 * n + 0, 8 * n + 2, 8 * n + 1,
755 3, 8 * n + 1, 8 * n + 2, 8 * n + 3,
756 3, 8 * n + 0, 8 * n + 4, 8 * n + 2,
757 3, 8 * n + 2, 8 * n + 4, 8 * n + 6,
758 3, 8 * n + 2, 8 * n + 6, 8 * n + 3,
759 3, 8 * n + 3, 8 * n + 6, 8 * n + 7,
760 3, 8 * n + 1, 8 * n + 5, 8 * n + 3,
761 3, 8 * n + 3, 8 * n + 5, 8 * n + 7,
762 3, 8 * n + 0, 8 * n + 4, 8 * n + 1,
763 3, 8 * n + 1, 8 * n + 4, 8 * n + 5,
764 3, 8 * n + 4, 8 * n + 6, 8 * n + 5,
765 3, 8 * n + 5, 8 * n + 6, 8 * n + 7};
767 unstructuredGrid->InsertNextCell(VTK_POLYHEDRON, 8, pointIds, 12,
771 vtkIdType pointIds[8] = {8 * n + 0, 8 * n + 1, 8 * n + 2, 8 * n + 3,
772 8 * n + 4, 8 * n + 5, 8 * n + 6, 8 * n + 7};
774 auto faces = vtkSmartPointer<vtkCellArray>::New();
776 vtkIdType f0[3] = {8 * n + 0, 8 * n + 2, 8 * n + 1};
777 vtkIdType f1[3] = {8 * n + 1, 8 * n + 2, 8 * n + 3};
779 vtkIdType f2[3] = {8 * n + 0, 8 * n + 4, 8 * n + 2};
780 vtkIdType f3[3] = {8 * n + 2, 8 * n + 4, 8 * n + 6};
782 vtkIdType f4[3] = {8 * n + 2, 8 * n + 6, 8 * n + 3};
783 vtkIdType f5[3] = {8 * n + 3, 8 * n + 6, 8 * n + 7};
785 vtkIdType f6[3] = {8 * n + 1, 8 * n + 5, 8 * n + 3};
786 vtkIdType f7[3] = {8 * n + 3, 8 * n + 5, 8 * n + 7};
788 vtkIdType f8[3] = {8 * n + 0, 8 * n + 4, 8 * n + 1};
789 vtkIdType f9[3] = {8 * n + 1, 8 * n + 4, 8 * n + 5};
791 vtkIdType fa[3] = {8 * n + 4, 8 * n + 6, 8 * n + 5};
792 vtkIdType fb[3] = {8 * n + 5, 8 * n + 6, 8 * n + 7};
794 faces->InsertNextCell(3, f0);
795 faces->InsertNextCell(3, f1);
796 faces->InsertNextCell(3, f2);
797 faces->InsertNextCell(3, f3);
798 faces->InsertNextCell(3, f4);
799 faces->InsertNextCell(3, f5);
800 faces->InsertNextCell(3, f6);
801 faces->InsertNextCell(3, f7);
802 faces->InsertNextCell(3, f8);
803 faces->InsertNextCell(3, f9);
804 faces->InsertNextCell(3, fa);
805 faces->InsertNextCell(3, fb);
807 unstructuredGrid->InsertNextCell(VTK_POLYHEDRON, 8, pointIds, 12,
808 faces->GetPointer());
811 n, global_vertices[n * (nVertices * balancer->getDimension() + 1) +
812 8 * balancer->getDimension()]);
813 cell->SetValue(n, (T)n);
815 unstructuredGrid->GetCellData()->AddArray(work);
816 unstructuredGrid->GetCellData()->AddArray(cell);
818 createDirectory(
"vtk_vertices");
819 std::ostringstream filename;
820 filename <<
"vtk_vertices/ALL_vtk_vertices_" << std::setw(7)
821 << std::setfill(
'0') << step <<
".vtu";
822 auto writer = vtkSmartPointer<vtkXMLUnstructuredGridWriter>::New();
823 writer->SetInputData(unstructuredGrid);
824 writer->SetFileName(filename.str().c_str());
825 writer->SetDataModeToAscii();