Coverage for rfpy/model/project.py: 100%

1from datetime import datetime

2from rfpy.model.exc import QuestionnaireStructureException

3from typing import Iterable, List, Sequence, Union

5from sqlalchemy import (Column, Unicode, Boolean, text, func, ForeignKeyConstraint,

6 Integer, ForeignKey, DateTime, Table, UniqueConstraint, Index)

7from sqlalchemy.orm import relationship, joinedload, deferred

8from sqlalchemy.dialects.mysql import TINYINT, VARCHAR, LONGTEXT, CHAR, INTEGER, MEDIUMTEXT

9from sqlalchemy.orm.session import object_session

10from sqlalchemy.orm.exc import NoResultFound

11import sqlalchemy.types as types

13from rfpy.auth import AuthorizationFailure

16from rfpy.model.meta import Base, AttachmentMixin, Visitor

17from rfpy.model.issue import Issue, IssueAttachment

18from rfpy.model.notify import ProjectWatchList

19from rfpy.model.questionnaire import (HierarchyWeightingsVisitor,

20 SaveTotalWeightingsVisitor,

21 LoadWeightSetVisitor,

22 WeightingSet,

23 Section)

25from rfpy.model import ProjectPermission, Participant, User, Organisation

27status = {

28 0: 'Draft',

29 10: 'Live',

30 20: 'Closed'

31}

32status_int = {v: k for k, v in status.items()}

35class Status(types.TypeDecorator):

37 impl = TINYINT(display_width=4)

39 cache_ok = True

41 def process_bind_param(self, value, dialect):

42 return status_int[value]

44 def process_result_value(self, value, dialect):

45 return status[value]

48class LazyParticipants(object):

50 """

51 Provides convenient set/collection syntax access for querying

52 participants associated with the given project

53 """

55 def __init__(self, project):

56 self.project = project

58 @property

59 def roleq(self):

60 return (self.project

61 .participants_query

62 .options(

63 joinedload('custom_role').load_only('name'),

64 joinedload('organisation').load_only('name', 'public'),

65 ))

67 def __contains__(self, organisation: Union[str, Organisation]):

68 org_id = organisation if isinstance(organisation, str) else organisation.id

69 q = self.project.participants_query.filter_by(org_id=org_id)

70 return object_session(self.project).query(q.exists()).scalar()

72 def __iter__(self):

73 for p in self.roleq:

74 yield p

76 def __len__(self):

77 return self.project.participants_query.count()

79 def get(self, org_id):

80 return self.project.participants_query.filter_by(org_id=org_id).one()

82 def append(self, participant):

83 return self.project.participants_query.append(participant)

85 def clear(self):

86 return self.project.participants_query.delete(synchronize_session='fetch')

89class LazyRestrictedUsers:

91 def __init__(self, project):

92 self.project = project

94 def project_perms_query(self):

95 session = object_session(self.project)

96 q = session.query(ProjectPermission)

97 q = q.join(Participant, Project)\

98 .filter(Participant.project == self.project)

99 return q

100

101 def __contains__(self, user):

102 q = self.project_perms_query()\

103 .filter(ProjectPermission.user == user,

104 Participant.organisation == user.organisation)

105 return object_session(self.project).query(q.exists()).scalar()

106

107 def __iter__(self):

108 for pp in self.project_perms_query():

109 yield pp

110

111

112proj_cat_rel = Table(

113 'project_categories',

114 Base.metadata,

115 Column('id', Integer, primary_key=True),

116 Column('project_id', Integer, index=True, nullable=False, server_default=text("'0'")),

117 Column('category_id', Integer, nullable=False, server_default=text("'0'")),

118 ForeignKeyConstraint(

119 ['category_id'],

120 ['categories.id'],

121 name='project_categories_ibfk_1',

122 ondelete='CASCADE'

123 ),

124 ForeignKeyConstraint(['project_id'], ['projects.id'], name='project_categories_ibfk_2'),

125 UniqueConstraint('project_id', 'category_id', name='project_category_unique'),

126 Index('category_id', 'category_id', 'project_id'),

127 mysql_engine='InnoDB',

128 mysql_charset='utf8mb4'

129)

130

131

132class Project(Base):

133 __tablename__ = 'projects'

134

135 __mapper_args__ = {

136 'polymorphic_on': 'status',

137 'polymorphic_identity': 'Base'

138 }

139

140 base_attrs = ('title,id,status,author_id,description,allow_private_communication,' +

141 'deadline,date_published,deadline,multiscored,url,maximum_score').split(',')

142

143 section_id = Column(

144 Integer,

145 ForeignKey(

146 'sections.id',

147 use_alter=True,

148 name="fk_prj_sec_id",

149 ondelete='SET NULL'

150 )

151 )

152

153 status = Column(Status, default='Draft', nullable=False)

154 org_id = Column(

155 VARCHAR(length=50),

156 ForeignKey('organisations.id', onupdate='CASCADE', ondelete='SET NULL'),

157 nullable=True)

158

159 author_id = Column(

160 VARCHAR(length=50),

161 ForeignKey('users.id', ondelete='SET NULL'),

162 nullable=True

163 )

164

165 allow_private_communication = Column(

166 TINYINT(display_width=4), nullable=False, server_default=text("'0'")

167 )

168 date_created = Column(DateTime, nullable=False, server_default=func.now())

169 date_published = Column(DateTime)

170 deadline = Column(DateTime)

171 description = deferred(Column(LONGTEXT()))

172 email = Column(Unicode(255))

173

174 # Some organisations want to prevent restricted users

175 # (domain experts) from accessing Respondent's attachment

176 expose_response_attachments = Column(

177 TINYINT(4), nullable=False, server_default=text("'0'")

178 )

179

180 # Whether Respondents should get to see weightings

181 expose_weightings = Column(

182 TINYINT(4), nullable=False, server_default=text("'0'")

183 )

184

185 # Don't allow buyers/consultants to view answers in Submitted Issues

186 # until the deadline of the Project has passed - some regulations demand

187 # that all bidders are evaluated over the same period of time.

188 hide_responses = Column(TINYINT(1), nullable=False, server_default=text("'0'"))

189

190 library = Column(TINYINT(1), nullable=False, server_default=text("'0'"))

191 maximum_score = Column(Integer, default=10, server_default=text("'10'"))

192 multiscored = Column(TINYINT(1), nullable=False, server_default=text("'0'"))

193 normalised_weights = Column(TINYINT(1), nullable=False, server_default=text("'0'"))

194 public = Column(TINYINT(4), nullable=False, server_default=text("'0'"))

195 questionnaire_locked = Column(TINYINT(1), nullable=False, server_default=text("'0'"))

196 reminder = Column(Integer)

197 require_acceptance = Column(TINYINT(1), nullable=False, server_default=text("'1'"))

198 revision = Column(Integer, nullable=False, server_default=text("'0'"))

199 template = Column(TINYINT(1), nullable=False, server_default=text("'0'"))

200 title = Column(Unicode(200), nullable=False, server_default=text("''"))

201 _type = Column(

202 'type', CHAR(4), nullable=False, default='norm', server_default=text("'norm'")

203 )

204 website = Column(VARCHAR(255))

205 lock_issues = Column(TINYINT(4), nullable=False, server_default=text("'0'"))

206

207 owner_org = relationship('Organisation', lazy="joined", uselist=False, backref='projects')

208

209 author = relationship('User', backref='authored_projects')

210

211 _issues = relationship('Issue', lazy='dynamic')

212

213 participant_watchers = relationship(

214 'User',

215 secondary='project_watch_list',

216 lazy='dynamic',

217 viewonly=True

218 )

219

220 questions = relationship('QuestionInstance', lazy='dynamic', viewonly=True)

221 participants_query = relationship('Participant', lazy='dynamic',

222 back_populates='project',

223 cascade='all, delete', passive_deletes=True)

224

225 root_section = relationship('Section', uselist=False, post_update=True,

226 primaryjoin="foreign(Project.section_id)==remote(Section.id)")

227

228 # If this is omitted here but declared as a backref on 'Section' then things start

229 # failing.

230 sections = relationship('Section', lazy="dynamic",

231 primaryjoin="Project.id==Section.project_id")

232

233 _attachments = relationship('ProjectAttachment', lazy='dynamic',

234 order_by='ProjectAttachment.position')

235

236 categories = relationship('Category', secondary=proj_cat_rel)

237

238 answer_reports = relationship('AnswerReport', back_populates='project',

239 cascade='all,delete', passive_deletes=True)

240

241 weighting_sets = relationship('WeightingSet', lazy='dynamic',

242 back_populates='project',

243 cascade='all,delete', passive_deletes=True)

244

245 project_fields = relationship('ProjectField', back_populates='project',

246 cascade='all,delete,delete-orphan', passive_deletes=True,

247 order_by='ProjectField.position')

248

249 qelements = relationship(

250 'QElement',

251 primaryjoin='Project.id==QuestionInstance.project_id',

252 secondaryjoin='QuestionDefinition.id==QElement.question_id',

253 secondary='join(QuestionInstance, QuestionDefinition)',

254 lazy='dynamic', viewonly=True

255 )

256

257 all_respondent_watchers = relationship(

258 User,

259 primaryjoin='Project.id==Issue.project_id',

260 secondaryjoin='IssueWatchList.user_id==User.id',

261 secondary='join(Issue, IssueWatchList)',

262 lazy='dynamic',

263 viewonly=True

264 )

265

266 def __init__(self, title, *args, **kwargs):

267 super(Project, self).__init__(*args, **kwargs)

268 self.title = title

269

270 @property

271 def issues(self) -> List[Issue]:

272 return self._issues.all()

273

274 def issue_by_id(self, issue_id):

275 return self._issues.filter_by(id=issue_id).one()

276

277 def question_by_number(self, question_number):

278

279 return self.questions\

280 .filter_by(number=question_number)\

281 .one()

282

283 @property

284 def published_issues(self) -> List[Issue]:

285 '''Issues visible to respondents'''

286 return self._issues\

287 .filter(Issue.status.in_(('Accepted', 'Submitted', 'Opportunity', 'Updateable')))\

288 .all()

289

290 @property

291 def opportunity_issues(self) -> List[Issue]:

292 '''Issues at status Opportunity'''

293 return self._issues\

294 .filter(Issue.status.in_(('Opportunity',)))\

295 .all()

296

297 @property

298 def respondent_watchers(self) -> Iterable[User]:

299 '''

300 A SQLAlchemy query of issue watchers whose issues are visible to Respondents

301 '''

302 visible_statuses = ('Accepted', 'Submitted', 'Opportunity', 'Updateable')

303 return self.all_respondent_watchers.filter(Issue.status.in_(visible_statuses))

304

305 def iter_all_watchers(self) -> Iterable[User]:

306 '''

307 Iterator returns all valid respondent or participant Users watching this project

308 '''

309 yield from self.respondent_watchers

310 yield from self.participant_watchers

311

312 @property

313 def deadline_passed(self):

314 if not self.deadline:

315 return False

316 return datetime.now() > self.deadline

317

318 @property

319 def scoreable_issues(self) -> List[Issue]:

320 if not hasattr(self, '_scoreable_issues_cache'):

321 issues = self\

322 ._issues\

323 .filter(Issue.scoreable_filter(self))\

324 .all()

325 self._scoreable_issues_cache = issues

326

327 return self._scoreable_issues_cache

328

329 def get_issue(self, issue_id) -> Issue:

330 '''

331 Fetch the Issue by id.

332

333 Raises NotFoundError if an Issue with the given ID doesn't belong to this project

334 '''

335 return self._issues.filter(Issue.id == issue_id).one()

336

337 def __repr__(self):

338 return '<Project %s : %s>' % (self.id, self.title)

339

340 @property

341 def participants(self) -> Sequence[Participant]:

342 '''Returns a collection which lazily implements Set operations'''

343 return LazyParticipants(self)

344

345 @property

346 def restricted_users(self):

347 return LazyRestrictedUsers(self)

348

349 @property

350 def project_type(self):

351 if self._type == 'gcpy':

352 return 'REFERENCE'

353 else:

354 return 'NORMAL'

355

356 @project_type.setter

357 def project_type(self, ptype):

358 if ptype not in ('REFERENCE', 'NORMAL'):

359 raise ValueError(f"{ptype} is not a recognised value for project_type")

360 if ptype == 'REFERENCE':

361 self._type = 'gcpy'

362 else:

363 self._type = 'norm'

364

365 def participant_role_permissions(self, user):

366 try:

367 participant = self.participants.get(user.organisation.id)

368 return participant.role_permissions

369 except NoResultFound:

370 return set()

371

372 def list_attachments(self, user):

373 self.check_attachments_visible(user)

374 return self._attachments.all()

375

376 def get_attachment(self, user, attachment_id):

377 self.check_attachments_visible(user)

378 return self._attachments.filter(ProjectAttachment.id == attachment_id).one()

379

380 def list_issue_attachments(self, user):

381 self.check_attachments_visible(user)

382 rows = self._issues\

383 .filter(Issue.scoreable_filter(self))\

384 .add_entity(IssueAttachment)\

385 .join(IssueAttachment)\

386 .order_by(Issue.id)\

387 .all()

388

389 return [att for _iss, att in rows]

390

391 def check_attachments_visible(self, user):

392 '''

393 Check if user can view attachments for this project

394 @raises AuthorizationFailure

395 '''

396 if user.is_restricted and not self.expose_response_attachments:

397 raise AuthorizationFailure('Attachments not visible to restricted users')

398

399 def get_participant(self, organisation):

400 '''

401 Fetch the Participant associated with the given organisation

402 for this project

403 @raise NoResultFound if organisation is not a participant

404 '''

405 return self.participants.get(organisation.id)

406

407 def query_visible_questions(self, user):

408 '''

409 Returns a Question query object, filtered by section permission

410 if the user is restricted

411 '''

412 if not user.is_restricted:

413 return self.questions

414 else:

415 return self.questions.join('section', 'perms').filter_by(user=user)

416

417 def visit_questionnaire(self, *visitors):

418 '''

419 Run the provided visitor(s) on the root section of this project.

420 '''

421 from rfpy.utils import benchmark

422 q = self.sections.options(joinedload('subsections')

423 .subqueryload('questions')

424 .lazyload('question_def')

425 .lazyload('elements'))

426

427 # Should be the first item returned by genexp

428 root = next(s for s in q.order_by(Section.parent_id) if s.parent_id is None)

429

430 if root.parent_id is not None or root.id != self.section_id:

431 m = f"Failed to find root section for project {self.id}"

432 raise QuestionnaireStructureException(m)

433

434 for visitor in visitors:

435 with benchmark('visitor %s' % visitor):

436 root.accept(visitor)

437 visitor.finalise()

438

439 return root

440

441 def print_questionnaire(self): # pragma: no cover

442 self.visit_questionnaire(PrintVisitor())

443

444 def calculate_normalised_weights(self):

445 self.visit_questionnaire(HierarchyWeightingsVisitor())

446

447 def save_total_weights(self, weighting_set_id=None):

448 sess = object_session(self)

449

450 if weighting_set_id:

451 ws = sess.query(WeightingSet).get(weighting_set_id)

452 visitors = (LoadWeightSetVisitor(ws),

453 HierarchyWeightingsVisitor(),

454 SaveTotalWeightingsVisitor(sess, self, weighting_set_id))

455 else:

456 visitors = (HierarchyWeightingsVisitor(),

457 SaveTotalWeightingsVisitor(sess, self, weighting_set_id))

458

459 self.visit_questionnaire(*visitors)

460

461 def delete_total_weights(self, weighting_set_id=None):

462 '''Delete TotalWeight records for this project for the given weighting set'''

463 self.total_weightings.filter_by(weighting_set_id=weighting_set_id).delete()

464

465 def total_weights_exist_for(self, weighting_set_id=None):

466 q = self.total_weightings\

467 .filter_by(weighting_set_id=weighting_set_id)

468 return q.count() > 0

469

470 def add_issue(self, issue):

471 self._issues.append(issue)

472

473 def add_watcher(self, user: User) -> bool:

474 '''

475 Adds the given user to this watch list. Returns True if successful,

476 False if the user is already watching

477 '''

478 if self.watch_list.filter_by(user=user).count() == 0:

479 self.watch_list.append(ProjectWatchList(user=user))

480 return True

481 else:

482 return False

483

484 def contains_section_id(self, section_id):

485 return self.sections.filter_by(id=section_id).count() == 1

486

487

488class DraftProject(Project):

489 __mapper_args__ = {

490 'polymorphic_identity': 'Draft'

491 }

492

493

494class LiveProject(Project):

495

496 __mapper_args__ = {

497 'polymorphic_identity': 'Live'

498 }

499

500 def build_score_key(self, issue, question, scoreset_id):

501 return "%s:%s:%s" % (issue.id, question.id, scoreset_id)

502

503 def generate_autoscores(self, session, user):

504 autoscores_dict = {}

505 for question in self.query_visible_questions(user):

506 if not question.is_autoscored:

507 continue

508 autoscore_map = question.calculate_autoscores()

509 for issue in self.scoreable_issues:

510 if issue in autoscore_map:

511 score_value = autoscore_map[issue]

512 key = self.build_score_key(issue, question, user.id)

513 autoscores_dict[key] = {'issue_id': issue.id,

514 'respondent': issue.respondent.name,

515 'question_id': question.id,

516 'scoreset_id': user.id,

517 'score': score_value,

518 'question_number': question.number.dotted}

519 return autoscores_dict

520

521 def scores_dict(self):

522 project_scores = {}

523 for issue in self.scoreable_issues:

524 for score in issue.scores.all():

525 key = self.build_score_key(issue, score.question, score.scoreset_id)

526 project_scores[key] = score

527 return project_scores

528

529

530class ClosedProject(Project):

531 __mapper_args__ = {

532 'polymorphic_identity': 'Closed'

533 }

534

535

536class ProjectAttachment(AttachmentMixin, Base):

537 __tablename__ = 'project_attachments'

538

539 public_attrs = ('id,description,size,filename,mimetype,private').split(',')

540 description = Column(VARCHAR(255))

541 project_id = Column(Integer, ForeignKey('projects.id', ondelete='SET NULL'))

542 date_uploaded = Column(DateTime, server_default=text('CURRENT_TIMESTAMP'))

543 author_id = Column(

544 VARCHAR(150),

545 ForeignKey('users.id', ondelete='SET NULL'),

546 nullable=True

547 )

548 org_id = Column(

549 VARCHAR(150),

550 ForeignKey('organisations.id', onupdate='CASCADE', ondelete='SET NULL'),

551 nullable=True

552 )

553 private = Column(Boolean, default=False, server_default=text('0'))

554 position = Column('pos', Integer, server_default=text('0'))

555 note_id = Column(Integer)

556

557 project = relationship(Project, back_populates="_attachments")

558 organisation = relationship('Organisation')

559

560

561class ProjectField(Base):

562 __tablename__ = 'project_fields'

563 __table_args__ = (

564 UniqueConstraint('project_id', 'key', name='project_field_id_key'),

565 UniqueConstraint('project_id', 'position', name='project_field_id_position'),

566 {'mysql_engine': "InnoDB", 'mysql_charset': 'utf8mb4'}

567 )

568 public_attrs = ['id', 'key', 'value', 'private']

569

570 project_id = Column(INTEGER(11), ForeignKey('projects.id', ondelete='CASCADE'), nullable=False)

571 private = Column(Boolean, default=True, nullable=False)

572 key = Column(VARCHAR(64), nullable=False)

573 value = Column(MEDIUMTEXT())

574 position = Column(Integer, nullable=False, default=0)

575

576 project = relationship('Project', back_populates='project_fields')

577

578 def __repr__(self) -> str:

579 return (f'<ProjectField#{self.id}: key={self.key}, '

580 f'position={self.position}, project_id={self.project_id}>')

581

582

583class PrintVisitor(Visitor): # pragma: no cover

584

585 '''Prints out the questionnaire structure for easier debugging'''

586

587 def __init__(self):

588 self.depth = 1

589

590 @property

591 def inset(self):

592 return ' ' * self.depth

593

594 def hello_section(self, sec):

595 print('')

596 fmt = '{inset} {title} - {num}/{id}/{pid}'\

597 + ' W: ({weight}, Norm: {norm_weight:.3f}, Abs: {abs_weight:.3f})'

598 normalised_weight = getattr(sec, 'normalised_weight', 0)

599 absolute_weight = getattr(sec, 'absolute_weight', 0)

600 sec_id = getattr(sec, 'id', '<no id set>')

601

602 out = fmt.format(inset=self.inset, num=sec.safe_number, title=sec.title,

603 weight=sec.weight, norm_weight=normalised_weight,

604 abs_weight=absolute_weight, id=sec_id, pid=sec.parent_id)

605 print(out)

606 self.depth += 1

607

608 def goodbye_section(self, _sec):

609 self.depth -= 1

610

611 def visit_question(self, q):

612 normalised_weight = getattr(q, 'normalised_weight', 0)

613 absolute_weight = getattr(q, 'absolute_weight', 0)

614 m = (f"{self.inset} Q: {q.safe_number} (W: {q.weight:.3f},"

615 f" Norm: {normalised_weight:.3f} Abs: {absolute_weight:.4f})")

616 print(m)